Control-flow Enforcement: Shadow Stack
Subsystems (20)
v14 cover Control-flow Enforcement: Shadow Stack
Control-flow Enforcement (CET) is a new Intel processor feature that blocks return/jump-oriented programming attacks. Details are in "Intel 64 and IA-32 Architectures Software Developer's Manual" [1].
CET can protect applications and the kernel. This series enables only application-level protection, and has three parts:
- Shadow stack [2],
- Indirect branch tracking [3], and
- Selftests [4].
I have run tests on these patches for quite some time, and they have been very stable. Linux distributions with CET are available now, and Intel processors with CET are becoming available. It would be nice if CET support can be accepted into the kernel. I will be working to address any issues should they come up.
Changes in v14:
- Update patch #10, add cpu_feature_enabled() to fixup_dirty_*.
- Update patch #19, instead of re-introducing do_mmap_pgoff(), make all callers of the wrapper pass a zero vm_flags.
- Update patch #26, move checking vm_flags into arch_validate_prot(); check the task's shadow stack status, instead of static_cpu_has().
[1] Intel 64 and IA-32 Architectures Software Developer's Manual:
https://software.intel.com/en-us/download/intel-64-and-ia-32- architectures-sdm-combined-volumes-1-2a-2b-2c-2d-3a-3b-3c-3d-and-4
[2] CET Shadow Stack patches v13:
https://lkml.kernel.org/r/[email protected]/
[3] Indirect Branch Tracking patches v13.
https://lkml.kernel.org/r/[email protected]/
[4] I am holding off the selftests changes and working to get Acked-by's. The earlier version of the selftests patches:
https://lkml.kernel.org/r/[email protected]/
[5] The kernel ptrace patch is tested with an Intel-internal updated GDB. I am holding off the kernel ptrace patch to re-test it with my earlier patch for fixing regset holes.
Yu-cheng Yu (26):
Documentation/x86: Add CET description x86/cpufeatures: Add CET CPU feature flags for Control-flow
Enforcement Technology (CET)
x86/fpu/xstate: Introduce CET MSR XSAVES supervisor states x86/cet: Add control-protection fault handler x86/cet/shstk: Add Kconfig option for user-mode Shadow Stack x86/mm: Change _PAGE_DIRTY to _PAGE_DIRTY_HW
x86/mm: Remove _PAGE_DIRTY_HW from kernel RO pages x86/mm: Introduce _PAGE_COW
drm/i915/gvt: Change _PAGE_DIRTY to _PAGE_DIRTY_BITS x86/mm: Update pte_modify for _PAGE_COW
x86/mm: Update ptep_set_wrprotect() and pmdp_set_wrprotect() for
transition from _PAGE_DIRTY_HW to _PAGE_COW
mm: Introduce VM_SHSTK for shadow stack memory x86/mm: Shadow Stack page fault error checking x86/mm: Update maybe_mkwrite() for shadow stack
mm: Fixup places that call pte_mkwrite() directly
mm: Add guard pages around a shadow stack.
mm/mmap: Add shadow stack pages to memory accounting
mm: Update can_follow_write_pte() for shadow stack
mm: Re-introduce vm_flags to do_mmap()
x86/cet/shstk: User-mode shadow stack support x86/cet/shstk: Handle signals for shadow stack
binfmt_elf: Define GNU_PROPERTY_X86_FEATURE_1_AND properties
ELF: Introduce arch_setup_elf_property() x86/cet/shstk: Handle thread shadow stack x86/cet/shstk: Add arch_prctl functions for shadow stack
mm: Introduce PROT_SHSTK for shadow stack
.../admin-guide/kernel-parameters.txt | 6 + Documentation/x86/index.rst | 1 + Documentation/x86/intel_cet.rst | 133 +++++++ arch/arm64/include/asm/elf.h | 5 + arch/x86/Kconfig | 39 ++ arch/x86/ia32/ia32_signal.c | 17 + arch/x86/include/asm/cet.h | 42 +++ arch/x86/include/asm/cpufeatures.h | 2 + arch/x86/include/asm/disabled-features.h | 8 +- arch/x86/include/asm/elf.h | 13 + arch/x86/include/asm/fpu/internal.h | 10 + arch/x86/include/asm/fpu/types.h | 23 +- arch/x86/include/asm/fpu/xstate.h | 5 +- arch/x86/include/asm/idtentry.h | 4 + arch/x86/include/asm/mman.h | 83 +++++ arch/x86/include/asm/mmu_context.h | 3 + arch/x86/include/asm/msr-index.h | 20 + arch/x86/include/asm/pgtable.h | 209 ++++++++++- arch/x86/include/asm/pgtable_types.h | 58 ++- arch/x86/include/asm/processor.h | 15 + arch/x86/include/asm/special_insns.h | 32 ++ arch/x86/include/asm/traps.h | 2 + arch/x86/include/uapi/asm/mman.h | 28 +- arch/x86/include/uapi/asm/prctl.h | 4 + arch/x86/include/uapi/asm/processor-flags.h | 2 + arch/x86/include/uapi/asm/sigcontext.h | 9 + arch/x86/kernel/Makefile | 2 + arch/x86/kernel/cet.c | 343 ++++++++++++++++++ arch/x86/kernel/cet_prctl.c | 68 ++++ arch/x86/kernel/cpu/common.c | 28 ++ arch/x86/kernel/cpu/cpuid-deps.c | 2 + arch/x86/kernel/fpu/signal.c | 100 +++++ arch/x86/kernel/fpu/xstate.c | 28 +- arch/x86/kernel/idt.c | 4 + arch/x86/kernel/process.c | 14 +- arch/x86/kernel/process_64.c | 32 ++ arch/x86/kernel/relocate_kernel_64.S | 2 +- arch/x86/kernel/signal.c | 10 + arch/x86/kernel/signal_compat.c | 2 +- arch/x86/kernel/traps.c | 59 +++ arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/mm/fault.c | 19 + arch/x86/mm/mmap.c | 2 + arch/x86/mm/pat/set_memory.c | 2 +- arch/x86/mm/pgtable.c | 25 ++ drivers/gpu/drm/i915/gvt/gtt.c | 2 +- fs/aio.c | 2 +- fs/binfmt_elf.c | 4 + fs/proc/task_mmu.c | 3 + include/linux/elf.h | 6 + include/linux/mm.h | 38 +- include/linux/pgtable.h | 35 ++ include/uapi/asm-generic/siginfo.h | 3 +- include/uapi/linux/elf.h | 9 + ipc/shm.c | 2 +- mm/gup.c | 8 +- mm/huge_memory.c | 10 +- mm/memory.c | 5 +- mm/migrate.c | 3 +- mm/mmap.c | 23 +- mm/mprotect.c | 2 +- mm/nommu.c | 4 +- mm/util.c | 2 +- scripts/as-x86_64-has-shadow-stack.sh | 4 + tools/arch/x86/include/asm/cpufeatures.h | 2 + .../arch/x86/include/asm/disabled-features.h | 8 +- tools/arch/x86/include/asm/msr-index.h | 20 + tools/arch/x86/include/uapi/asm/prctl.h | 4 + 68 files changed, 1624 insertions(+), 92 deletions(-) create mode 100644 Documentation/x86/intel_cet.rst create mode 100644 arch/x86/include/asm/cet.h create mode 100644 arch/x86/include/asm/mman.h create mode 100644 arch/x86/kernel/cet.c create mode 100644 arch/x86/kernel/cet_prctl.c create mode 100755 scripts/as-x86_64-has-shadow-stack.sh
Signature · 1 line
2.21.0
PATCH v14 1/26 Documentation/x86: Add CET description
Explain no_user_shstk/no_user_ibt kernel parameters, and introduce a new document on Control-flow Enforcement Technology (CET).
--- .../admin-guide/kernel-parameters.txt | 6 + Documentation/x86/index.rst | 1 + Documentation/x86/intel_cet.rst | 133 ++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 Documentation/x86/intel_cet.rst
Documentation/admin-guide/kernel-parameters.txt +6 -0
@@ -3164,6 +3164,12 @@ noexec=on: enable non-executable mappings (default) noexec=off: disable non-executable mappings + no_user_shstk [X86-64] Disable Shadow Stack for user-mode + applications + + no_user_ibt [X86-64] Disable Indirect Branch Tracking for user-mode + applications + nosmap [X86,PPC] Disable SMAP (Supervisor Mode Access Prevention) even if it is supported by processor.
Documentation/x86/index.rst +1 -0
@@ -19,6 +19,7 @@ x86-specific Documentation tlb mtrr pat + intel_cet intel-iommu intel_txt amd-memory-encryption
Documentation/x86/intel_cet.rst new file +133 -0
133 lines not on this page · raw message
Signature · 1 line
2.21.0
Borislav Petkov
maintainerRe: [PATCH v14 01/26] Documentation/x86: Add CET description
Documentation/x86/intel_cet.rst line 18
Quoted 12 lines · Yu-cheng Yu
+[1] Overview
+============Documentation/x86/intel_cet.rst new file +10
-0+ +Control-flow Enforcement Technology (CET) is an Intel processor feature +that provides protection against return/jump-oriented programming (ROP) +attacks. It can be set up to protect both applications and the kernel. +Only user-mode protection is implemented in the 64-bit kernel, including +support for running legacy 32-bit applications. + +CET introduces Shadow Stack and Indirect Branch Tracking. Shadow stack is +a secondary stack allocated from memory and cannot be directly modified by +applications. When executing a CALL, the processor pushes the return
^ . .. instruction ...
Quoted 21 lines
Documentation/x86/intel_cet.rst new file +21
-0+address to both the normal stack and the shadow stack. Upon function +return, the processor pops the shadow stack copy and compares it to the +normal stack copy. If the two differ, the processor raises a control- +protection fault. Indirect branch tracking verifies indirect CALL/JMP +targets are intended as marked by the compiler with 'ENDBR' opcodes. + +There are two kernel configuration options: + + X86_SHADOW_STACK_USER, and + X86_BRANCH_TRACKING_USER. + +These need to be enabled to build a CET-enabled kernel, and Binutils v2.31 +and GCC v8.1 or later are required to build a CET kernel. To build a CET- +enabled application, GLIBC v2.28 or later is also required. + +There are two command-line options for disabling CET features:: + + no_user_shstk - disables user shadow stack, and + no_user_ibt - disables user indirect branch tracking. + +At run time, /proc/cpuinfo shows CET features if the processor supports
Documentation/x86/intel_cet.rst line 49
Quoted 10 lines
Documentation/x86/intel_cet.rst new file +10
-0+CET. + +[2] Application Enabling +======================== + +An application's CET capability is marked in its ELF header and can be +verified from the following command output, in the NT_GNU_PROPERTY_TYPE_0 +field: + + readelf -n <application>
Can be verified how? What does it say for a CET-enabled executable? Put it here in the doc pls.
Documentation/x86/intel_cet.rst line 61
Quoted 12 lines
Documentation/x86/intel_cet.rst new file +12
-0+ +If an application supports CET and is statically linked, it will run with +CET protection. If the application needs any shared libraries, the loader +checks all dependencies and enables CET when all requirements are met. + +[3] Backward Compatibility +========================== + +GLIBC provides a few tunables for backward compatibility. + +GLIBC_TUNABLES=glibc.tune.hwcaps=-SHSTK,-IBT + Turn off SHSTK/IBT for the current shell.
For the current shell? How?
You mean, you execute the kernel shell with that variable set? So you set this variable in any executable's env which links with glibc in order to disable CET?
In any case, this needs clarification.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH v14 01/26] Documentation/x86: Add CET description
Quoted 16 lines · Borislav Petkov
Documentation/x86/intel_cet.rst line 18
Quoted 12 lines · Yu-cheng Yu
+[1] Overview
+============Documentation/x86/intel_cet.rst new file +10
-0+ +Control-flow Enforcement Technology (CET) is an Intel processor feature +that provides protection against return/jump-oriented programming (ROP) +attacks. It can be set up to protect both applications and the kernel. +Only user-mode protection is implemented in the 64-bit kernel, including +support for running legacy 32-bit applications. + +CET introduces Shadow Stack and Indirect Branch Tracking. Shadow stack is +a secondary stack allocated from memory and cannot be directly modified by +applications. When executing a CALL, the processor pushes the return^ . .. instruction ...
I will update it.
[...]
Quoted 4 lines
Quoted 4 lines
Documentation/x86/intel_cet.rst new file +4
-0+ +[2] Application Enabling +======================== +
Quoted 9 lines
Documentation/x86/intel_cet.rst line 49
Quoted 5 lines
Documentation/x86/intel_cet.rst new file +5
-0+An application's CET capability is marked in its ELF header and can be +verified from the following command output, in the NT_GNU_PROPERTY_TYPE_0 +field: + + readelf -n <application>Can be verified how? What does it say for a CET-enabled executable? Put it here in the doc pls.
readelf -n <application> | grep SHSTK
properties: x86 feature: IBT, SHSTK
I will add this.
[...]
Quoted 10 lines
Documentation/x86/intel_cet.rst line 61
Quoted 7 lines
+[3] Backward Compatibility
+==========================Documentation/x86/intel_cet.rst new file +5
-0+ +GLIBC provides a few tunables for backward compatibility. + +GLIBC_TUNABLES=glibc.tune.hwcaps=-SHSTK,-IBT + Turn off SHSTK/IBT for the current shell.For the current shell? How?
Quoted 6 lines
You mean, you execute the kernel shell with that variable set? So you set this variable in any executable's env which links with glibc in order to disable CET?
In any case, this needs clarification.
In the current shell, if GLIBC_TUNABLES variable is set as such, applications started will have CET features disabled. I can put more details here, or maybe a reference to the GLIBC man pages.
Thanks,
Yu-cheng
Borislav Petkov
maintainerRe: [PATCH v14 01/26] Documentation/x86: Add CET description
Quoted 3 lines · Yu-cheng
In the current shell, if
GLIBC_TUNABLESvariable is set as such, applications started will have CET features disabled. I can put more details here, or maybe a reference to the GLIBC man pages.
Why do you keep repeating "the current shell"? You pass it with setenv(3) too, can't you?
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH v14 01/26] Documentation/x86: Add CET description
Quoted 8 lines · Borislav Petkov
Quoted 3 lines · Yu-cheng
In the current shell, if
GLIBC_TUNABLESvariable is set as such, applications started will have CET features disabled. I can put more details here, or maybe a reference to the GLIBC man pages.Why do you keep repeating "the current shell"? You pass it with setenv(3) too, can't you?
Ture. I will revise it.
Yu-cheng
PATCH v14 2/26 x86/cpufeatures: Add CET CPU feature flags for Control-flow Enforcement Technology (CET)
In reply to Yu-cheng YuAdd CPU feature flags for Control-flow Enforcement Technology (CET).
CPUID.(EAX=7,ECX=0):ECX[bit 7] Shadow stack CPUID.(EAX=7,ECX=0):EDX[bit 20] Indirect Branch Tracking
--- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kernel/cpu/cpuid-deps.c | 2 ++ tools/arch/x86/include/asm/cpufeatures.h | 2 ++ 3 files changed, 6 insertions(+)
arch/x86/include/asm/cpufeatures.h +2 -0
@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */
@@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */
arch/x86/kernel/cpu/cpuid-deps.c +2 -0
@@ -69,6 +69,8 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL }, + { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES }, + { X86_FEATURE_IBT, X86_FEATURE_XSAVES }, {} };
tools/arch/x86/include/asm/cpufeatures.h +2 -0
@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */
@@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */
Signature · 1 line
2.21.0
Borislav Petkov
maintainerRe: [PATCH v14 02/26] x86/cpufeatures: Add CET CPU feature flags for Control-flow Enforcement Technology (CET)
Quoted 7 lines · Yu-cheng Yu
Add CPU feature flags for Control-flow Enforcement Technology (CET).
CPUID.(EAX=7,ECX=0):ECX[bit 7] Shadow stack CPUID.(EAX=7,ECX=0):EDX[bit 20] Indirect Branch Tracking
Signed-off-by Yu-cheng Yu <[email protected]>Reviewed-by Borislav Petkov <[email protected]>
This is not the patch I reviewed, why do you keep my Reviewed-by tag?
Quoted 53 lines
Reviewed-by Kees Cook <[email protected]>--- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kernel/cpu/cpuid-deps.c | 2 ++ tools/arch/x86/include/asm/cpufeatures.h | 2 ++ 3 files changed, 6 insertions(+)arch/x86/include/asm/cpufeatures.h +2
-0@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */ @@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */ arch/x86/kernel/cpu/cpuid-deps.c +2
-0@@ -69,6 +69,8 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL }, + { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES }, + { X86_FEATURE_IBT, X86_FEATURE_XSAVES }, {} };tools/arch/x86/include/asm/cpufeatures.h +1
-0@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */
tools/arch/x86/include/asm/cpufeatures.h line 377
Quoted 7 lines
tools/arch/x86/include/asm/cpufeatures.h +1
-0@@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */
We don't sync the respective change in tools/ - Arnaldo does.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH v14 02/26] x86/cpufeatures: Add CET CPU feature flags for Control-flow Enforcement Technology (CET)
Quoted 10 lines · Borislav Petkov
Quoted 7 lines · Yu-cheng Yu
Add CPU feature flags for Control-flow Enforcement Technology (CET).
CPUID.(EAX=7,ECX=0):ECX[bit 7] Shadow stack CPUID.(EAX=7,ECX=0):EDX[bit 20] Indirect Branch Tracking
Signed-off-by Yu-cheng Yu <[email protected]>Reviewed-by Borislav Petkov <[email protected]>This is not the patch I reviewed, why do you keep my Reviewed-by tag?
I will drop it. It has been re-based many times, and probably I accidentally introduced something else?
Quoted 55 lines
Quoted 55 lines
Reviewed-by Kees Cook <[email protected]>--- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kernel/cpu/cpuid-deps.c | 2 ++ tools/arch/x86/include/asm/cpufeatures.h | 2 ++ 3 files changed, 6 insertions(+)arch/x86/include/asm/cpufeatures.h +2
-0@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */ @@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */ arch/x86/kernel/cpu/cpuid-deps.c +2
-0@@ -69,6 +69,8 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC }, { X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL }, + { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES }, + { X86_FEATURE_IBT, X86_FEATURE_XSAVES }, {} };tools/arch/x86/include/asm/cpufeatures.h +1
-0@@ -341,6 +341,7 @@#define X86_FEATURE_OSPKE (16*32+ 4)/* OS Protection Keys Enable */ #define X86_FEATURE_WAITPKG (16*32+ 5)/* UMONITOR/UMWAIT/TPAUSE Instructions */ #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6)/* Additional AVX512 Vector Bit Manipulation Instructions */ +#define X86_FEATURE_SHSTK (16*32+ 7)/* Shadow Stack */ #define X86_FEATURE_GFNI (16*32+ 8)/* Galois Field New Instructions */ #define X86_FEATURE_VAES (16*32+ 9)/* Vector AES */ #define X86_FEATURE_VPCLMULQDQ (16*32+10)/* Carry-Less Multiplication Double Quadword */ @@ -370,6 +371,7 @@#define X86_FEATURE_SERIALIZE (18*32+14)/* SERIALIZE instruction */
Quoted 8 lines
tools/arch/x86/include/asm/cpufeatures.h line 377
Quoted 6 lines
tools/arch/x86/include/asm/cpufeatures.h +1
-0#define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */ #define X86_FEATURE_FLUSH_L1D (18*32+28)/* Flush L1D cache */ We don't sync the respective change in
tools/- Arnaldo doe
Got it. I will remove this.
Yu-cheng
Borislav Petkov
maintainerRe: [PATCH v14 02/26] x86/cpufeatures: Add CET CPU feature flags for Control-flow Enforcement Technology (CET)
Quoted 2 lines · Yu-cheng
I will drop it. It has been re-based many times, and probably I accidentally introduced something else?
Yah, I think I added my tag to this version:
https://lkml.kernel.org/lkml/[email protected]/
Do you need to refresh on when tags get dropped?
See here: Documentation/process/submitting-patches.rst
You should verify the rest of the patchset too - tags are not sticked to a patch forever.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH v14 02/26] x86/cpufeatures: Add CET CPU feature flags for Control-flow Enforcement Technology (CET)
Quoted 6 lines · Borislav Petkov
Quoted 2 lines · Yu-cheng
I will drop it. It has been re-based many times, and probably I accidentally introduced something else?
Yah, I think I added my tag to this version:
Quoted 9 lines
https://lkml.kernel.org/lkml/[email protected]/
Do you need to refresh on when tags get dropped?
See here:
Documentation/process/submitting-patches.rstYou should verify the rest of the patchset too - tags are not sticked to a patch forever.
I will do that.
Yu-cheng
PATCH v14 3/26 x86/fpu/xstate: Introduce CET MSR XSAVES supervisor states
In reply to Yu-cheng YuControl-flow Enforcement Technology (CET) adds five MSRs. Introduce them and their XSAVES supervisor states:
MSR_IA32_U_CET (user-mode CET settings), MSR_IA32_PL3_SSP (user-mode Shadow Stack pointer), MSR_IA32_PL0_SSP (kernel-mode Shadow Stack pointer), MSR_IA32_PL1_SSP (Privilege Level 1 Shadow Stack pointer), MSR_IA32_PL2_SSP (Privilege Level 2 Shadow Stack pointer).
--- arch/x86/include/asm/fpu/types.h | 23 +++++++++++++++-- arch/x86/include/asm/fpu/xstate.h | 5 ++-- arch/x86/include/asm/msr-index.h | 20 +++++++++++++++ arch/x86/include/uapi/asm/processor-flags.h | 2 ++ arch/x86/kernel/fpu/xstate.c | 28 ++++++++++++++++++--- tools/arch/x86/include/asm/msr-index.h | 20 +++++++++++++++ 6 files changed, 91 insertions(+), 7 deletions(-)
arch/x86/include/asm/fpu/types.h +21 -2
@@ -115,8 +115,8 @@ enum xfeature {XFEATURE_PT_UNIMPLEMENTED_SO_FAR ,XFEATURE_PKRU ,XFEATURE_RSRVD_COMP_10 , -XFEATURE_RSRVD_COMP_11 , -XFEATURE_RSRVD_COMP_12 , +XFEATURE_CET_USER , +XFEATURE_CET_KERNEL ,XFEATURE_RSRVD_COMP_13 ,XFEATURE_RSRVD_COMP_14 , XFEATURE_LBR,
@@ -134,6 +134,8 @@ enum xfeature {#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM)#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) +#define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) +#define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL)#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
@@ -236,6 +238,23 @@ struct pkru_state {u32 pad; } __packed; +/* +* State component 11 is Control-flow Enforcement user states +*/ +struct cet_user_state { +u64 user_cet;/* user control-flow settings */ +u64 user_ssp;/* user shadow stack pointer */ +}; + +/* +* State component 12 is Control-flow Enforcement kernel states +*/ +struct cet_kernel_state { +u64 kernel_ssp;/* kernel shadow stack */ +u64 pl1_ssp;/* privilege level 1 shadow stack */ +u64 pl2_ssp;/* privilege level 2 shadow stack */ +}; +/* * State component 15: Architectural LBR configuration state. * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
arch/x86/include/asm/fpu/xstate.h +3 -2
@@ -35,7 +35,7 @@ XFEATURE_MASK_BNDCSR)/* All currently supported supervisor features */ -#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (0) +#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_CET_USER)/* * A supervisor state component may not always contain valuable information,
@@ -62,7 +62,8 @@* Unsupported supervisor features. When a supervisor feature in this mask is * supported in the future, move it to the supported supervisor feature mask. */ -#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT) +#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT | \ + XFEATURE_MASK_CET_KERNEL)/* All supervisor states including supported and unsupported states. */ #define XFEATURE_MASK_SUPERVISOR_ALL (XFEATURE_MASK_SUPERVISOR_SUPPORTED | \
arch/x86/include/asm/msr-index.h +20 -0
@@ -912,4 +912,24 @@#define MSR_VM_IGNNE 0xc0010115#define MSR_VM_HSAVE_PA 0xc0010117 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x6a0/* user mode cet setting */ +#define MSR_IA32_S_CET 0x6a2/* kernel mode cet setting */ +#define MSR_IA32_PL0_SSP 0x6a4/* kernel shstk pointer */ +#define MSR_IA32_PL1_SSP 0x6a5/* ring-1 shstk pointer */ +#define MSR_IA32_PL2_SSP 0x6a6/* ring-2 shstk pointer */ +#define MSR_IA32_PL3_SSP 0x6a7/* user shstk pointer */ +#define MSR_IA32_INT_SSP_TAB 0x6a8/* exception shstk table */ + +/* MSR_IA32_U_CET and MSR_IA32_S_CET bits */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) +#endif /* _ASM_X86_MSR_INDEX_H */
arch/x86/include/uapi/asm/processor-flags.h +2 -0
@@ -130,6 +130,8 @@#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)#define X86_CR4_PKE_BIT 22/* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_CET_BIT 23/* enable Control-flow Enforcement */ +#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)/* * x86-64 Task Priority Register, CR8
arch/x86/kernel/fpu/xstate.c +25 -3
@@ -38,6 +38,9 @@ static const char *xfeature_names[] ="Processor Trace (unused)" ,"Protection Keys User registers" ,"unknown xstate feature" , +"Control-flow User registers" , +"Control-flow Kernel registers" , + "unknown xstate feature" , }; static short xsave_cpuid_features[] __initdata = {
@@ -51,6 +54,9 @@ static short xsave_cpuid_features[] __initdata = {X86_FEATURE_AVX512F ,X86_FEATURE_INTEL_PT ,X86_FEATURE_PKU , +-1 ,/* Unused */ +X86_FEATURE_SHSTK ,/* XFEATURE_CET_USER */ + X86_FEATURE_SHSTK, /* XFEATURE_CET_KERNEL */ };/*
@@ -318,6 +324,8 @@ static void __init print_xstate_features(void)print_xstate_feature (XFEATURE_MASK_ZMM_Hi256);print_xstate_feature (XFEATURE_MASK_Hi16_ZMM);print_xstate_feature (XFEATURE_MASK_PKRU ); +print_xstate_feature (XFEATURE_MASK_CET_USER ); +print_xstate_feature (XFEATURE_MASK_CET_KERNEL ); }/*
@@ -592,6 +600,8 @@ static void check_xstate_against_struct(int nr) XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state); XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); + XCHECK_SZ(sz, nr, XFEATURE_CET_USER, struct cet_user_state); + XCHECK_SZ(sz, nr, XFEATURE_CET_KERNEL, struct cet_kernel_state);/* * Make *SURE* to add any feature numbers in below if
@@ -601,7 +611,8 @@ static void check_xstate_against_struct(int nr)if ((nr <XFEATURE_YMM ) || (nr >=XFEATURE_MAX ) || (nr ==XFEATURE_PT_UNIMPLEMENTED_SO_FAR ) || - ((nr >=XFEATURE_RSRVD_COMP_10 ) && (nr <=XFEATURE_LBR ))) { + (nr ==XFEATURE_RSRVD_COMP_10 ) || + ((nr >=XFEATURE_RSRVD_COMP_13 ) && (nr <=XFEATURE_LBR ))) {WARN_ONCE (1 ,"no structure for xstate: %d\n" , nr);XSTATE_WARN_ON (1 ); }
@@ -831,8 +842,19 @@ void __init fpu__init_system_xstate(void)* Clear XSAVE features that are disabled in the normal CPUID. */ for (i =0 ; i <ARRAY_SIZE (xsave_cpuid_features); i++) { -if (!boot_cpu_has (xsave_cpuid_features[i])) - xfeatures_mask_all &= ~BIT_ULL (i); +if (xsave_cpuid_features[i] ==X86_FEATURE_SHSTK ) { +/* +* X86_FEATURE_SHSTK and X86_FEATURE_IBT share +* same states, but can be enabled separately. +*/ +if (!boot_cpu_has (X86_FEATURE_SHSTK ) && + !boot_cpu_has (X86_FEATURE_IBT )) + xfeatures_mask_all &= ~BIT_ULL (i); + }else { +if ((xsave_cpuid_features[i] ==-1 ) || + !boot_cpu_has (xsave_cpuid_features[i])) + xfeatures_mask_all &= ~BIT_ULL (i); + } } xfeatures_mask_all &=fpu__get_supported_xfeatures_mask ();
tools/arch/x86/include/asm/msr-index.h +20 -0
@@ -912,4 +912,24 @@#define MSR_VM_IGNNE 0xc0010115#define MSR_VM_HSAVE_PA 0xc0010117 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x6a0/* user mode cet setting */ +#define MSR_IA32_S_CET 0x6a2/* kernel mode cet setting */ +#define MSR_IA32_PL0_SSP 0x6a4/* kernel shstk pointer */ +#define MSR_IA32_PL1_SSP 0x6a5/* ring-1 shstk pointer */ +#define MSR_IA32_PL2_SSP 0x6a6/* ring-2 shstk pointer */ +#define MSR_IA32_PL3_SSP 0x6a7/* user shstk pointer */ +#define MSR_IA32_INT_SSP_TAB 0x6a8/* exception shstk table */ + +/* MSR_IA32_U_CET and MSR_IA32_S_CET bits */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) +#endif /* _ASM_X86_MSR_INDEX_H */
Signature · 1 line
2.21.0
Re: [PATCH v14 03/26] x86/fpu/xstate: Introduce CET MSR XSAVES supervisor states
On Mon, Oct 12, 2020 at 08:38:27AM -0700, Yu-cheng Yu wrote:
...
Quoted 17 lines
/*
* x86-64 Task Priority Register, CR8arch/x86/kernel/fpu/xstate.c +3
-0@@ -38,6 +38,9 @@ static const char *xfeature_names[] ="Processor Trace (unused)" ,"Protection Keys User registers" ,"unknown xstate feature" , +"Control-flow User registers" , +"Control-flow Kernel registers" , + "unknown xstate feature" , }; static short xsave_cpuid_features[] __initdata = {
arch/x86/kernel/fpu/xstate.c static short xsave_cpuid_features[] __initdata = { · line 60
Quoted 7 lines
arch/x86/kernel/fpu/xstate.c +3
-0@@ -51,6 +54,9 @@ static short xsave_cpuid_features[] __initdata = {X86_FEATURE_AVX512F ,X86_FEATURE_INTEL_PT ,X86_FEATURE_PKU , +-1 ,/* Unused */ +X86_FEATURE_SHSTK ,/* XFEATURE_CET_USER */ + X86_FEATURE_SHSTK, /* XFEATURE_CET_KERNEL */ };
Why do you need "-1" here in the array? The only 1:1 mapping is between the names itselves and values, not indices of arrays so i don't understand why we need this unused value. Sorry if it is a dumb questions and been discussed already.
Re: [PATCH v14 03/26] x86/fpu/xstate: Introduce CET MSR XSAVES supervisor states
Quoted 17 lines · Cyrill Gorcunov
On Mon, Oct 12, 2020 at 08:38:27AM -0700, Yu-cheng Yu wrote:
...Quoted 15 lines
/*
* x86-64 Task Priority Register, CR8arch/x86/kernel/fpu/xstate.c +3
-0@@ -38,6 +38,9 @@ static const char *xfeature_names[] ="Processor Trace (unused)" ,"Protection Keys User registers" ,"unknown xstate feature" , +"Control-flow User registers" , +"Control-flow Kernel registers" , + "unknown xstate feature" , };
Quoted 15 lines
arch/x86/kernel/fpu/xstate.c static short xsave_cpuid_features[] __initdata = { · line 60
Quoted 9 lines
arch/x86/kernel/fpu/xstate.c +3
-0static short xsave_cpuid_features[]__initdata = {@@ -51,6 +54,9 @@ static short xsave_cpuid_features[] __initdata = {X86_FEATURE_AVX512F ,X86_FEATURE_INTEL_PT ,X86_FEATURE_PKU , +-1 ,/* Unused */ +X86_FEATURE_SHSTK ,/* XFEATURE_CET_USER */ + X86_FEATURE_SHSTK, /* XFEATURE_CET_KERNEL */ };Why do you need "-1" here in the array? The only 1:1 mapping is between the names itselves and values, not indices of arrays so i don't understand why we need this unused value. Sorry if it is a dumb questions and been discussed already.
The indices are used indirectly in fpu__init_system_xstate() to set bits in xfeatures_mask_all, i.e.
xfeatures_mask_all &= ~BIT_ULL(i).
So they need to match the xstate feature bits.
Yu-cheng
PATCH v14 4/26 x86/cet: Add control-protection fault handler
In reply to Yu-cheng YuA control-protection fault is triggered when a control-flow transfer attempt violates Shadow Stack or Indirect Branch Tracking constraints. For example, the return address for a RET instruction differs from the copy on the Shadow Stack; or an indirect JMP instruction, without the NOTRACK prefix, arrives at a non-ENDBR opcode.
The control-protection fault handler works in a similar way as the general protection fault handler. It provides the si_code SEGV_CPERR to the signal handler.
--- arch/x86/include/asm/idtentry.h | 4 ++ arch/x86/kernel/idt.c | 4 ++ arch/x86/kernel/signal_compat.c | 2 +- arch/x86/kernel/traps.c | 59 ++++++++++++++++++++++++++++++ include/uapi/asm-generic/siginfo.h | 3 +- 5 files changed, 70 insertions(+), 2 deletions(-)
arch/x86/include/asm/idtentry.h +4 -0
@@ -532,6 +532,10 @@ DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment);DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_GP , exc_general_protection);DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_AC , exc_alignment_check); +#ifdef CONFIG_X86_CET +DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_CP , exc_control_protection); +#endif +/* Raw exception entries which need extra work */ DECLARE_IDTENTRY_RAW (X86_TRAP_UD , exc_invalid_op);DECLARE_IDTENTRY_RAW (X86_TRAP_BP , exc_int3);
arch/x86/kernel/idt.c +4 -0
@@ -112,6 +112,10 @@ static const __initconst struct idt_data def_idts[] = {#elif defined(CONFIG_X86_32 ) SYSG(IA32_SYSCALL_VECTOR, entry_INT80_32),#endif + +#ifdef CONFIG_X86_CET + INTG(X86_TRAP_CP, asm_exc_control_protection), +#endif };/*
arch/x86/kernel/signal_compat.c +1 -1
@@ -27,7 +27,7 @@ static inline void signal_compat_build_tests(void)*/ BUILD_BUG_ON (NSIGILL !=11 );BUILD_BUG_ON (NSIGFPE !=15 ); -BUILD_BUG_ON (NSIGSEGV !=7 ); +BUILD_BUG_ON (NSIGSEGV !=8 );BUILD_BUG_ON (NSIGBUS !=5 );BUILD_BUG_ON (NSIGTRAP !=5 );BUILD_BUG_ON (NSIGCHLD !=6 );
arch/x86/kernel/traps.c +59 -0
65 lines not on this page · raw message
include/uapi/asm-generic/siginfo.h +2 -1
@@ -229,7 +229,8 @@ typedef struct siginfo {#define SEGV_ACCADI 5/* ADI not enabled for mapped object */ #define SEGV_ADIDERR 6/* Disrupting MCD error */ #define SEGV_ADIPERR 7/* Precise MCD exception */ -#define NSIGSEGV 7 +#define SEGV_CPERR 8/* Control protection fault */ +#define NSIGSEGV 8/* * SIGBUS si_codes
Signature · 1 line
2.21.0
PATCH v14 5/26 x86/cet/shstk: Add Kconfig option for user-mode Shadow Stack
In reply to Yu-cheng YuShadow Stack provides protection against function return address corruption. It is active when the processor supports it, the kernel has CONFIG_X86_SHADOW_STACK_USER, and the application is built for the feature. This is only implemented for the 64-bit kernel. When it is enabled, legacy non-shadow stack applications continue to work, but without protection.
--- arch/x86/Kconfig | 33 +++++++++++++++++++++++++++ scripts/as-x86_64-has-shadow-stack.sh | 4 ++++ 2 files changed, 37 insertions(+) create mode 100755 scripts/as-x86_64-has-shadow-stack.sh
arch/x86/Kconfig +33 -0
@@ -1927,6 +1927,39 @@ config X86_INTEL_TSX_MODE_AUTO side channel attacks- equals the tsx=auto command line parameter. endchoice +config AS_HAS_SHADOW_STACK + def_bool $(success,$(srctree)/scripts/as-x86_64-has-shadow-stack.sh $(CC)) + help + Test the assembler for shadow stack instructions. + +config X86_CET + def_bool n + +config ARCH_HAS_SHADOW_STACK + def_bool n + +config X86_SHADOW_STACK_USER + prompt "Intel Shadow Stacks for user-mode" + def_bool n + depends on CPU_SUP_INTEL && X86_64 + depends on AS_HAS_SHADOW_STACK + select ARCH_USES_HIGH_VMA_FLAGS + select X86_CET + select ARCH_HAS_SHADOW_STACK + help + Shadow Stacks provides protection against program stack + corruption. It's a hardware feature. This only matters + if you have the right hardware. It's a security hardening + feature and apps must be enabled to use it. You get no + protection "for free" on old userspace. The hardware can + support user and kernel, but this option is for user space + only. + Support for this feature is only known to be present on + processors released in 2020 or later. CET features are also + known to increase kernel text size by 3.7 KB. + + If unsure, say N. + config EFI bool "EFI runtime service support" depends on ACPI
scripts/as-x86_64-has-shadow-stack.sh new file +4 -0
@@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +echo "wrussq %rax, (%rbx)" | $* -x assembler -c -
Signature · 1 line
2.21.0
PATCH v14 6/26 x86/mm: Change _PAGE_DIRTY to _PAGE_DIRTY_HW
In reply to Yu-cheng YuBefore introducing _PAGE_COW for non-hardware memory management purposes in the next patch, rename _PAGE_DIRTY to _PAGE_DIRTY_HW and _PAGE_BIT_DIRTY to _PAGE_BIT_DIRTY_HW to make meanings more clear. There are no functional changes from this patch.
--- arch/x86/include/asm/pgtable.h | 18 +++++++++--------- arch/x86/include/asm/pgtable_types.h | 10 +++++----- arch/x86/kernel/relocate_kernel_64.S | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-)
arch/x86/include/asm/pgtable.h +9 -9
@@ -124,7 +124,7 @@ extern pmdval_t early_pmd_flags;*/ static inline int pte_dirty (pte_t pte) { -return pte_flags (pte) & _PAGE_DIRTY; +return pte_flags (pte) & _PAGE_DIRTY_HW; }
@@ -163,7 +163,7 @@ static inline int pte_young(pte_t pte)static inline int pmd_dirty (pmd_t pmd) { -return pmd_flags (pmd) & _PAGE_DIRTY; +return pmd_flags (pmd) & _PAGE_DIRTY_HW; }static inline int pmd_young (pmd_t pmd)
@@ -173,7 +173,7 @@ static inline int pmd_young(pmd_t pmd)static inline int pud_dirty (pud_t pud) { -return pud_flags (pud) & _PAGE_DIRTY; +return pud_flags (pud) & _PAGE_DIRTY_HW; }static inline int pud_young (pud_t pud)
@@ -334,7 +334,7 @@ static inline pte_t pte_clear_uffd_wp(pte_t pte)static inline pte_t pte_mkclean (pte_t pte) { -return pte_clear_flags (pte, _PAGE_DIRTY); +return pte_clear_flags (pte, _PAGE_DIRTY_HW); }static inline pte_t pte_mkold (pte_t pte)
@@ -354,7 +354,7 @@ static inline pte_t pte_mkexec(pte_t pte)static inline pte_t pte_mkdirty (pte_t pte) { -return pte_set_flags (pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); +return pte_set_flags (pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); }static inline pte_t pte_mkyoung (pte_t pte)
@@ -435,7 +435,7 @@ static inline pmd_t pmd_mkold(pmd_t pmd)static inline pmd_t pmd_mkclean (pmd_t pmd) { -return pmd_clear_flags (pmd, _PAGE_DIRTY); +return pmd_clear_flags (pmd, _PAGE_DIRTY_HW); }static inline pmd_t pmd_wrprotect (pmd_t pmd)
@@ -445,7 +445,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd)static inline pmd_t pmd_mkdirty (pmd_t pmd) { -return pmd_set_flags (pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); +return pmd_set_flags (pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); }static inline pmd_t pmd_mkdevmap (pmd_t pmd)
@@ -489,7 +489,7 @@ static inline pud_t pud_mkold(pud_t pud)static inline pud_t pud_mkclean (pud_t pud) { -return pud_clear_flags (pud, _PAGE_DIRTY); +return pud_clear_flags (pud, _PAGE_DIRTY_HW); }static inline pud_t pud_wrprotect (pud_t pud)
@@ -499,7 +499,7 @@ static inline pud_t pud_wrprotect(pud_t pud)static inline pud_t pud_mkdirty (pud_t pud) { -return pud_set_flags (pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); +return pud_set_flags (pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); }static inline pud_t pud_mkdevmap (pud_t pud)
arch/x86/include/asm/pgtable_types.h +5 -5
@@ -15,7 +15,7 @@#define _PAGE_BIT_PWT 3/* page write through */ #define _PAGE_BIT_PCD 4/* page cache disabled */ #define _PAGE_BIT_ACCESSED 5/* was accessed (raised by CPU) */ -#define _PAGE_BIT_DIRTY 6/* was written to (raised by CPU) */ +#define _PAGE_BIT_DIRTY_HW 6/* was written to (raised by CPU) */ #define _PAGE_BIT_PSE 7/* 4 MB (or 2MB) page */ #define _PAGE_BIT_PAT 7/* on 4KB pages */ #define _PAGE_BIT_GLOBAL 8/* Global TLB entry PPro+ */
@@ -46,7 +46,7 @@#define _PAGE_PWT (_AT(pteval_t, 1) << _PAGE_BIT_PWT)#define _PAGE_PCD (_AT(pteval_t, 1) << _PAGE_BIT_PCD)#define _PAGE_ACCESSED (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED) -#define _PAGE_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY) +#define _PAGE_DIRTY_HW (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY_HW)#define _PAGE_PSE (_AT(pteval_t, 1) << _PAGE_BIT_PSE)#define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL)#define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1)
@@ -74,7 +74,7 @@ _PAGE_PKEY_BIT3)#if defined(CONFIG_X86_64 ) || defined(CONFIG_X86_PAE ) -#define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) +#define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY_HW | _PAGE_ACCESSED)#else #define _PAGE_KNL_ERRATUM_MASK 0#endif
@@ -126,7 +126,7 @@* pte_modify() does modify it. */ #define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ - _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ + _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY_HW | \ _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC | \ _PAGE_UFFD_WP)#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)
@@ -163,7 +163,7 @@ enum page_cache_mode {#define __RW _PAGE_RW#define _USR _PAGE_USER#define ___A _PAGE_ACCESSED -#define ___D _PAGE_DIRTY +#define ___D _PAGE_DIRTY_HW#define ___G _PAGE_GLOBAL#define __NX _PAGE_NX
arch/x86/kernel/relocate_kernel_64.S +1 -1
@@ -17,7 +17,7 @@ */ #define PTR(x) (x << 3) -#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) +#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY_HW) /* * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
arch/x86/kvm/vmx/vmx.c +1 -1
@@ -3620,7 +3620,7 @@ static int init_rmode_identity_map(struct kvm *kvm)/* Set up identity-mapping pagetable for EPT in real mode */ for (i =0 ; i <PT32_ENT_PER_PAGE ; i++) { tmp = (i <<22 ) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | - _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); + _PAGE_ACCESSED | _PAGE_DIRTY_HW | _PAGE_PSE); r =kvm_write_guest_page (kvm, identity_map_pfn, &tmp, i *sizeof (tmp),sizeof (tmp));if (r <0 )
Signature · 1 line
2.21.0
PATCH v14 7/26 x86/mm: Remove _PAGE_DIRTY_HW from kernel RO pages
In reply to Yu-cheng YuKernel read-only PTEs are setup as _PAGE_DIRTY_HW. Since these become shadow stack PTEs, remove the dirty bit.
--- arch/x86/include/asm/pgtable_types.h | 6 +++--- arch/x86/mm/pat/set_memory.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-)
arch/x86/include/asm/pgtable_types.h +3 -3
@@ -193,10 +193,10 @@ enum page_cache_mode {#define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC)#define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0)#define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC) -#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX|___D| 0|___G) -#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0|___D| 0|___G) +#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G) +#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G)#define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC) -#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX|___D| 0|___G) +#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G)#define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G)#define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G)#define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP)
arch/x86/mm/pat/set_memory.c +1 -1
@@ -1932,7 +1932,7 @@ int set_memory_nx(unsigned long addr, int numpages)int set_memory_ro (unsigned long addr,int numpages) { -return change_page_attr_clear (&addr, numpages,__pgprot (_PAGE_RW),0 ); +return change_page_attr_clear (&addr, numpages,__pgprot (_PAGE_RW | _PAGE_DIRTY_HW),0 ); }int set_memory_rw (unsigned long addr,int numpages)
Signature · 1 line
2.21.0
PATCH v14 8/26 x86/mm: Introduce _PAGE_COW
In reply to Yu-cheng YuThere is essentially no room left in the x86 hardware PTEs on some OSes (not Linux). That left the hardware architects looking for a way to represent a new memory type (shadow stack) within the existing bits. They chose to repurpose a lightly-used state: Write=0,Dirty=1.
The reason it's lightly used is that Dirty=1 is normally set by hardware and cannot normally be set by hardware on a Write=0 PTE. Software must normally be involved to create one of these PTEs, so software can simply opt to not create them.
But that leaves us with a Linux problem: we need to ensure we never create Write=0,Dirty=1 PTEs. In places where we do create them, we need to find an alternative way to represent them _without_ using the same hardware bit combination. Thus, enter _PAGE_COW. This results in the following:
(a) A modified, copy-on-write (COW) page: (R/O + _PAGE_COW)
(b) A R/O page that has been COW'ed: (R/O + _PAGE_COW)
The user page is in a R/O VMA, and get_user_pages() needs a writable copy. The page fault handler creates a copy of the page and sets the new copy's PTE as R/O and _PAGE_COW.
(c) A shadow stack PTE: (R/O + _PAGE_DIRTY_HW)
(d) A shared shadow stack PTE: (R/O + _PAGE_COW)
When a shadow stack page is being shared among processes (this happens at fork()), its PTE is cleared of _PAGE_DIRTY_HW, so the next shadow stack access causes a fault, and the page is duplicated and _PAGE_DIRTY_HW is set again. This is the COW equivalent for shadow stack pages, even though it's copy-on-access rather than copy-on-write.
(e) A page where the processor observed a Write=1 PTE, started a write, set Dirty=1, but then observed a Write=0 PTE. That's possible today, but will not happen on processors that support shadow stack.
Use _PAGE_COW in pte_wrprotect() and _PAGE_DIRTY_HW in pte_mkwrite(). Apply the same changes to pmd and pud.
When this patch is applied, there are six free bits left in the 64-bit PTE. There are no more free bits in the 32-bit PTE (except for PAE) and shadow stack is not implemented for the 32-bit kernel.
--- arch/x86/include/asm/pgtable.h | 120 ++++++++++++++++++++++++--- arch/x86/include/asm/pgtable_types.h | 41 ++++++++- 2 files changed, 150 insertions(+), 11 deletions(-)
arch/x86/include/asm/pgtable.h +110 -10
@@ -122,9 +122,9 @@ extern pmdval_t early_pmd_flags;* The following only work if pte_present() is true. * Undefined behaviour if not.. */ -static inline int pte_dirty (pte_t pte) +static inline bool pte_dirty (pte_t pte) { -return pte_flags (pte) & _PAGE_DIRTY_HW; +return pte_flags (pte) & _PAGE_DIRTY_BITS; }
@@ -161,9 +161,9 @@ static inline int pte_young(pte_t pte)return pte_flags (pte) & _PAGE_ACCESSED; } -static inline int pmd_dirty (pmd_t pmd) +static inline bool pmd_dirty (pmd_t pmd) { -return pmd_flags (pmd) & _PAGE_DIRTY_HW; +return pmd_flags (pmd) & _PAGE_DIRTY_BITS; }static inline int pmd_young (pmd_t pmd)
@@ -171,9 +171,9 @@ static inline int pmd_young(pmd_t pmd)return pmd_flags (pmd) & _PAGE_ACCESSED; } -static inline int pud_dirty (pud_t pud) +static inline bool pud_dirty (pud_t pud) { -return pud_flags (pud) & _PAGE_DIRTY_HW; +return pud_flags (pud) & _PAGE_DIRTY_BITS; }static inline int pud_young (pud_t pud)
@@ -183,6 +183,12 @@ static inline int pud_young(pud_t pud)static inline int pte_write (pte_t pte) { +/* +* If _PAGE_DIRTY_HW is set, the PTE must either have +* _PAGE_RW or be a shadow stack PTE, which is logically writable. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) +return pte_flags (pte) & (_PAGE_RW | _PAGE_DIRTY_HW);return pte_flags (pte) & _PAGE_RW; }
@@ -334,7 +340,7 @@ static inline pte_t pte_clear_uffd_wp(pte_t pte)static inline pte_t pte_mkclean (pte_t pte) { -return pte_clear_flags (pte, _PAGE_DIRTY_HW); +return pte_clear_flags (pte, _PAGE_DIRTY_BITS); }static inline pte_t pte_mkold (pte_t pte)
@@ -344,6 +350,17 @@ static inline pte_t pte_mkold(pte_t pte)static inline pte_t pte_wrprotect (pte_t pte) { +/* +* Blindly clearing _PAGE_RW might accidentally create +* a shadow stack PTE (RW=0,Dirty=1). Move the hardware +* dirty value to the software bit. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { + pte.pte |= (pte.pte & _PAGE_DIRTY_HW) >> + _PAGE_BIT_DIRTY_HW << _PAGE_BIT_COW; + pte =pte_clear_flags (pte, _PAGE_DIRTY_HW); + } +return pte_clear_flags (pte, _PAGE_RW); }
@@ -354,6 +371,18 @@ static inline pte_t pte_mkexec(pte_t pte)static inline pte_t pte_mkdirty (pte_t pte) { +pteval_t dirty = _PAGE_DIRTY_HW; + +/* Avoid creating (HW)Dirty=1,Write=0 PTEs */ +if (cpu_feature_enabled (X86_FEATURE_SHSTK ) && !pte_write (pte)) + dirty = _PAGE_COW; + +return pte_set_flags (pte, dirty | _PAGE_SOFT_DIRTY); +} + +static inline pte_t pte_mkwrite_shstk (pte_t pte) +{ + pte =pte_clear_flags (pte, _PAGE_COW);return pte_set_flags (pte, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); }
@@ -364,6 +393,13 @@ static inline pte_t pte_mkyoung(pte_t pte)static inline pte_t pte_mkwrite (pte_t pte) { +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +if (pte_flags (pte) & _PAGE_COW) { + pte =pte_clear_flags (pte, _PAGE_COW); + pte =pte_set_flags (pte, _PAGE_DIRTY_HW); + } + } +return pte_set_flags (pte, _PAGE_RW); }
@@ -435,16 +471,41 @@ static inline pmd_t pmd_mkold(pmd_t pmd)static inline pmd_t pmd_mkclean (pmd_t pmd) { -return pmd_clear_flags (pmd, _PAGE_DIRTY_HW); +return pmd_clear_flags (pmd, _PAGE_DIRTY_BITS); }static inline pmd_t pmd_wrprotect (pmd_t pmd) { +/* +* Blindly clearing _PAGE_RW might accidentally create +* a shadow stack PMD (RW=0,Dirty=1). Move the hardware +* dirty value to the software bit. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +pmdval_t v =native_pmd_val (pmd); + + v |= (v & _PAGE_DIRTY_HW) >> _PAGE_BIT_DIRTY_HW << + _PAGE_BIT_COW; + pmd =pmd_clear_flags (__pmd (v), _PAGE_DIRTY_HW); + } +return pmd_clear_flags (pmd, _PAGE_RW); }static inline pmd_t pmd_mkdirty (pmd_t pmd) { +pmdval_t dirty = _PAGE_DIRTY_HW; + +/* Avoid creating (HW)Dirty=1,Write=0 PMDs */ +if (cpu_feature_enabled (X86_FEATURE_SHSTK ) && !(pmd_flags (pmd) & _PAGE_RW)) + dirty = _PAGE_COW; + +return pmd_set_flags (pmd, dirty | _PAGE_SOFT_DIRTY); +} + +static inline pmd_t pmd_mkwrite_shstk (pmd_t pmd) +{ + pmd =pmd_clear_flags (pmd, _PAGE_COW);return pmd_set_flags (pmd, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); }
@@ -465,6 +526,13 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd)static inline pmd_t pmd_mkwrite (pmd_t pmd) { +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +if (pmd_flags (pmd) & _PAGE_COW) { + pmd =pmd_clear_flags (pmd, _PAGE_COW); + pmd =pmd_set_flags (pmd, _PAGE_DIRTY_HW); + } + } +return pmd_set_flags (pmd, _PAGE_RW); }
@@ -489,17 +557,36 @@ static inline pud_t pud_mkold(pud_t pud)static inline pud_t pud_mkclean (pud_t pud) { -return pud_clear_flags (pud, _PAGE_DIRTY_HW); +return pud_clear_flags (pud, _PAGE_DIRTY_BITS); }static inline pud_t pud_wrprotect (pud_t pud) { +/* +* Blindly clearing _PAGE_RW might accidentally create +* a shadow stack PUD (RW=0,Dirty=1). Move the hardware +* dirty value to the software bit. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +pudval_t v =native_pud_val (pud); + + v |= (v & _PAGE_DIRTY_HW) >> _PAGE_BIT_DIRTY_HW << + _PAGE_BIT_COW; + pud =pud_clear_flags (__pud (v), _PAGE_DIRTY_HW); + } +return pud_clear_flags (pud, _PAGE_RW); }static inline pud_t pud_mkdirty (pud_t pud) { -return pud_set_flags (pud, _PAGE_DIRTY_HW | _PAGE_SOFT_DIRTY); +pudval_t dirty = _PAGE_DIRTY_HW; + +/* Avoid creating (HW)Dirty=1,Write=0 PUDs */ +if (cpu_feature_enabled (X86_FEATURE_SHSTK ) && !(pud_flags (pud) & _PAGE_RW)) + dirty = _PAGE_COW; + +return pud_set_flags (pud, dirty | _PAGE_SOFT_DIRTY); }static inline pud_t pud_mkdevmap (pud_t pud)
@@ -519,6 +606,13 @@ static inline pud_t pud_mkyoung(pud_t pud)static inline pud_t pud_mkwrite (pud_t pud) { +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +if (pud_flags (pud) & _PAGE_COW) { + pud =pud_clear_flags (pud, _PAGE_COW); + pud =pud_set_flags (pud, _PAGE_DIRTY_HW); + } + } +return pud_set_flags (pud, _PAGE_RW); }
@@ -1132,6 +1226,12 @@ extern int pmdp_clear_flush_young(struct vm_area_struct *vma,#define pmd_write pmd_writestatic inline int pmd_write (pmd_t pmd) { +/* +* If _PAGE_DIRTY_HW is set, then the PMD must either have +* _PAGE_RW or be a shadow stack PMD, which is logically writable. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) +return pmd_flags (pmd) & (_PAGE_RW | _PAGE_DIRTY_HW);return pmd_flags (pmd) & _PAGE_RW; }
arch/x86/include/asm/pgtable_types.h +40 -1
59 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH v14 9/26 drm/i915/gvt: Change _PAGE_DIRTY to _PAGE_DIRTY_BITS
In reply to Yu-cheng YuAfter the introduction of _PAGE_COW, a modified page's PTE can have either _PAGE_DIRTY_HW or _PAGE_COW. Change _PAGE_DIRTY to _PAGE_DIRTY_BITS.
--- drivers/gpu/drm/i915/gvt/gtt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/i915/gvt/gtt.c +1 -1
@@ -1207,7 +1207,7 @@ static int split_2MB_gtt_entry(struct intel_vgpu *vgpu, }/* Clear dirty field. */ - se->val64 &= ~_PAGE_DIRTY; + se->val64 &= ~_PAGE_DIRTY_BITS; ops->clear_pse (se); ops->clear_ips (se);
Signature · 1 line
2.21.0
PATCH v14 10/26 x86/mm: Update pte_modify for _PAGE_COW
In reply to Yu-cheng YuPte_modify() changes a PTE to 'newprot'. It doesn't use the pte_*() helpers that a previous patch fixed up, so we need a new site.
Introduce fixup_dirty_pte() to set the dirty bits based on _PAGE_RW, and apply the same changes to pmd_modify().
--- arch/x86/include/asm/pgtable.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)
arch/x86/include/asm/pgtable.h +33 -0
@@ -727,6 +727,21 @@ static inline pmd_t pmd_mkinvalid(pmd_t pmd)static inline u64 flip_protnone_guard (u64 oldval,u64 val,u64 mask); +static inline pteval_t fixup_dirty_pte (pteval_t pteval) +{ +pte_t pte =__pte (pteval); + +if (cpu_feature_enabled (X86_FEATURE_SHSTK ) &&pte_dirty (pte)) { + pte =pte_mkclean (pte); + +if (pte_flags (pte) & _PAGE_RW) + pte =pte_set_flags (pte, _PAGE_DIRTY_HW); +else + pte =pte_set_flags (pte, _PAGE_COW); + } +return pte_val (pte); +} +static inline pte_t pte_modify (pte_t pte,pgprot_t newprot) {pteval_t val =pte_val (pte), oldval = val;
@@ -737,16 +752,34 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)*/ val &= _PAGE_CHG_MASK; val |=check_pgprot (newprot) & ~_PAGE_CHG_MASK; + val =fixup_dirty_pte (val); val =flip_protnone_guard (oldval, val,PTE_PFN_MASK );return __pte (val); } +static inline int pmd_write (pmd_t pmd); +static inline pmdval_t fixup_dirty_pmd (pmdval_t pmdval) +{ +pmd_t pmd =__pmd (pmdval); + +if (cpu_feature_enabled (X86_FEATURE_SHSTK ) &&pmd_dirty (pmd)) { + pmd =pmd_mkclean (pmd); + +if (pmd_flags (pmd) & _PAGE_RW) + pmd =pmd_set_flags (pmd, _PAGE_DIRTY_HW); +else + pmd =pmd_set_flags (pmd, _PAGE_COW); + } +return pmd_val (pmd); +} +static inline pmd_t pmd_modify (pmd_t pmd,pgprot_t newprot) {pmdval_t val =pmd_val (pmd), oldval = val; val &= _HPAGE_CHG_MASK; val |=check_pgprot (newprot) & ~_HPAGE_CHG_MASK; + val =fixup_dirty_pmd (val); val =flip_protnone_guard (oldval, val,PHYSICAL_PMD_PAGE_MASK );return __pmd (val); }
Signature · 1 line
2.21.0
PATCH v14 11/26 x86/mm: Update ptep_set_wrprotect() and pmdp_set_wrprotect() for transition from _PAGE_DIRTY_HW to _PAGE_COW
In reply to Yu-cheng YuWhen shadow stack is introduced, [R/O + _PAGE_DIRTY_HW] PTE is reserved for shadow stack. Copy-on-write PTEs have [R/O + _PAGE_COW].
When a PTE goes from [R/W + _PAGE_DIRTY_HW] to [R/O + _PAGE_COW], it could become a transient shadow stack PTE in two cases:
The first case is that some processors can start a write but end up seeing a read-only PTE by the time they get to the Dirty bit, creating a transient shadow stack PTE. However, this will not occur on processors supporting shadow stack, therefore we don't need a TLB flush here.
The second case is that when the software, without atomic, tests & replaces _PAGE_DIRTY_HW with _PAGE_COW, a transient shadow stack PTE can exist. This is prevented with cmpxchg.
Dave Hansen, Jann Horn, Andy Lutomirski, and Peter Zijlstra provided many insights to the issue. Jann Horn provided the cmpxchg solution.
--- arch/x86/include/asm/pgtable.h | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+)
arch/x86/include/asm/pgtable.h +52 -0
@@ -1230,6 +1230,32 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,static inline void ptep_set_wrprotect (struct mm_struct *mm,unsigned long addr,pte_t *ptep) { +/* +* Some processors can start a write, but end up seeing a read-only +* PTE by the time they get to the Dirty bit. In this case, they +* will set the Dirty bit, leaving a read-only, Dirty PTE which +* looks like a shadow stack PTE. +* +* However, this behavior has been improved and will not occur on +* processors supporting shadow stack. Without this guarantee, a +* transition to a non-present PTE and flush the TLB would be +* needed. +* +* When changing a writable PTE to read-only and if the PTE has +* _PAGE_DIRTY_HW set, move that bit to _PAGE_COW so that the +* PTE is not a shadow stack PTE. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +pte_t old_pte, new_pte; + +do { + old_pte =READ_ONCE (*ptep); + new_pte =pte_wrprotect (old_pte); + + }while (!try_cmpxchg (&ptep->pte, &old_pte.pte, new_pte.pte)); + +return ; + }clear_bit (_PAGE_BIT_RW, (unsigned long *)&ptep->pte); }
@@ -1286,6 +1312,32 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,static inline void pmdp_set_wrprotect (struct mm_struct *mm,unsigned long addr,pmd_t *pmdp) { +/* +* Some processors can start a write, but end up seeing a read-only +* PMD by the time they get to the Dirty bit. In this case, they +* will set the Dirty bit, leaving a read-only, Dirty PMD which +* looks like a Shadow Stack PMD. +* +* However, this behavior has been improved and will not occur on +* processors supporting Shadow Stack. Without this guarantee, a +* transition to a non-present PMD and flush the TLB would be +* needed. +* +* When changing a writable PMD to read-only and if the PMD has +* _PAGE_DIRTY_HW set, we move that bit to _PAGE_COW so that the +* PMD is not a shadow stack PMD. +*/ +if (cpu_feature_enabled (X86_FEATURE_SHSTK )) { +pmd_t old_pmd, new_pmd; + +do { + old_pmd =READ_ONCE (*pmdp); + new_pmd =pmd_wrprotect (old_pmd); + + }while (!try_cmpxchg ((pmdval_t *)pmdp, (pmdval_t *)&old_pmd,pmd_val (new_pmd))); + +return ; + }clear_bit (_PAGE_BIT_RW, (unsigned long *)pmdp); }
Signature · 1 line
2.21.0
PATCH v14 12/26 mm: Introduce VM_SHSTK for shadow stack memory
In reply to Yu-cheng YuA Shadow Stack PTE must be read-only and have _PAGE_DIRTY set. However, read-only and Dirty PTEs also exist for copy-on-write (COW) pages. These two cases are handled differently for page faults. Introduce VM_SHSTK to track shadow stack VMAs.
--- arch/x86/mm/mmap.c | 2 ++ fs/proc/task_mmu.c | 3 +++ include/linux/mm.h | 8 ++++++++ 3 files changed, 13 insertions(+)
arch/x86/mm/mmap.c +2 -0
@@ -165,6 +165,8 @@ unsigned long get_mmap_base(int is_legacy)const char *arch_vma_name (struct vm_area_struct *vma) { +if (vma->vm_flags &VM_SHSTK ) +return "[shadow stack]" ;return NULL ; }
fs/proc/task_mmu.c +3 -0
@@ -663,6 +663,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) [ilog2(VM_PKEY_BIT4)] = "",#endif #endif /* CONFIG_ARCH_HAS_PKEYS */ +#ifdef CONFIG_X86_SHADOW_STACK_USER + [ilog2(VM_SHSTK)] = "ss", +#endif };size_t i;
include/linux/mm.h +8 -0
@@ -304,11 +304,13 @@ extern unsigned int kobjsize(const void *objp);#define VM_HIGH_ARCH_BIT_2 34/* bit only usable on 64-bit architectures */ #define VM_HIGH_ARCH_BIT_3 35/* bit only usable on 64-bit architectures */ #define VM_HIGH_ARCH_BIT_4 36/* bit only usable on 64-bit architectures */ +#define VM_HIGH_ARCH_BIT_5 37/* bit only usable on 64-bit architectures */ #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)#define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4) +#define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5)#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */ #ifdef CONFIG_ARCH_HAS_PKEYS
@@ -324,6 +326,12 @@ extern unsigned int kobjsize(const void *objp);#endif #endif /* CONFIG_ARCH_HAS_PKEYS */ +#ifdef CONFIG_X86_SHADOW_STACK_USER +# define VM_SHSTK VM_HIGH_ARCH_5 +#else +# define VM_SHSTK VM_NONE +#endif +#if defined(CONFIG_X86 )# define VM_PAT VM_ARCH_1/* PAT reserves whole VMA at once (x86) */ #elif defined(CONFIG_PPC )
Signature · 1 line
2.21.0
PATCH v14 13/26 x86/mm: Shadow Stack page fault error checking
In reply to Yu-cheng YuShadow stack accesses are those that are performed by the CPU where it expects to encounter a shadow stack mapping. These accesses are performed implicitly by CALL/RET at the site of the shadow stack pointer. These accesses are made explicitly by shadow stack management instructions like WRUSSQ.
Shadow stacks accesses to shadow-stack mapping can see faults in normal, valid operation just like regular accesses to regular mappings. Shadow stacks need some of the same features like delayed allocation, swap and copy-on-write.
Shadow stack accesses can also result in errors, such as when a shadow stack overflows, or if a shadow stack access occurs to a non-shadow-stack mapping.
In handling a shadow stack page fault, verify it occurs within a shadow stack mapping. It is always an error otherwise. For valid shadow stack accesses, set FAULT_FLAG_WRITE to effect copy-on-write. Because clearing _PAGE_DIRTY_HW (vs. _PAGE_RW) is used to trigger the fault, shadow stack read fault and shadow stack write fault are not differentiated and both are handled as a write access.
--- arch/x86/include/asm/traps.h | 2 ++ arch/x86/mm/fault.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+)
arch/x86/include/asm/traps.h +2 -0
@@ -50,6 +50,7 @@ void __noreturn handle_stack_overflow(const char *message,* bit 3 == 1: use of reserved bit detected * bit 4 == 1: fault was an instruction fetch * bit 5 == 1: protection keys block access +* bit 6 == 1: shadow stack access fault */ enum x86_pf_error_code {X86_PF_PROT =1 <<0 ,
@@ -58,5 +59,6 @@ enum x86_pf_error_code {X86_PF_RSVD =1 <<3 ,X86_PF_INSTR =1 <<4 ,X86_PF_PK =1 <<5 , + X86_PF_SHSTK = 1 << 6, };#endif /* _ASM_X86_TRAPS_H */
arch/x86/mm/fault.c +19 -0
@@ -1110,6 +1110,17 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) (error_code & X86_PF_INSTR), foreign))return 1 ; +/* +* Verify a shadow stack access is within a shadow stack VMA. +* It is always an error otherwise. Normal data access to a +* shadow stack area is checked in the case followed. +*/ +if (error_code &X86_PF_SHSTK ) { +if (!(vma->vm_flags &VM_SHSTK )) +return 1 ; +return 0 ; + } +if (error_code &X86_PF_WRITE ) {/* write, present and write, not present: */ if (unlikely (!(vma->vm_flags &VM_WRITE )))
@@ -1275,6 +1286,14 @@ void do_user_addr_fault(struct pt_regs *regs,perf_sw_event (PERF_COUNT_SW_PAGE_FAULTS ,1 , regs, address); +/* +* Clearing _PAGE_DIRTY_HW is used to detect shadow stack access. +* This method cannot distinguish shadow stack read vs. write. +* For valid shadow stack accesses, set FAULT_FLAG_WRITE to effect +* copy-on-write. +*/ +if (hw_error_code &X86_PF_SHSTK ) + flags |=FAULT_FLAG_WRITE ;if (hw_error_code &X86_PF_WRITE ) flags |=FAULT_FLAG_WRITE ;if (hw_error_code &X86_PF_INSTR )
Signature · 1 line
2.21.0
PATCH v14 14/26 x86/mm: Update maybe_mkwrite() for shadow stack
In reply to Yu-cheng YuShadow stack memory is writable, but its VMA has VM_SHSTK instead of VM_WRITE. Update maybe_mkwrite() to include the shadow stack.
--- arch/x86/Kconfig | 4 ++++ arch/x86/mm/pgtable.c | 18 ++++++++++++++++++ include/linux/mm.h | 2 ++ include/linux/pgtable.h | 24 ++++++++++++++++++++++++ mm/huge_memory.c | 2 ++ 5 files changed, 50 insertions(+)
arch/x86/Kconfig +4 -0
@@ -1935,6 +1935,9 @@ config AS_HAS_SHADOW_STACK config X86_CET def_bool n +config ARCH_MAYBE_MKWRITE + def_bool n + config ARCH_HAS_SHADOW_STACK def_bool n
@@ -1945,6 +1948,7 @@ config X86_SHADOW_STACK_USER depends on AS_HAS_SHADOW_STACK select ARCH_USES_HIGH_VMA_FLAGS select X86_CET + select ARCH_MAYBE_MKWRITE select ARCH_HAS_SHADOW_STACK help Shadow Stacks provides protection against program stack
arch/x86/mm/pgtable.c +18 -0
@@ -610,6 +610,24 @@ int pmdp_clear_flush_young(struct vm_area_struct *vma, }#endif +#ifdef CONFIG_ARCH_MAYBE_MKWRITE +pte_t arch_maybe_mkwrite (pte_t pte,struct vm_area_struct *vma) +{ +if (likely (vma->vm_flags &VM_SHSTK )) + pte =pte_mkwrite_shstk (pte); +return pte; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +pmd_t arch_maybe_pmd_mkwrite (pmd_t pmd,struct vm_area_struct *vma) +{ +if (likely (vma->vm_flags &VM_SHSTK )) + pmd =pmd_mkwrite_shstk (pmd); +return pmd; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_ARCH_MAYBE_MKWRITE */ +/** * reserve_top_address - reserves a hole in the top of kernel address space * @reserve - size of hole to reserve
include/linux/mm.h +2 -0
@@ -969,6 +969,8 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) {if (likely (vma->vm_flags &VM_WRITE )) pte =pte_mkwrite (pte); +else + pte =arch_maybe_mkwrite (pte, vma);return pte; }
include/linux/pgtable.h +24 -0
@@ -1356,6 +1356,30 @@ static inline bool arch_has_pfn_modify_check(void) }#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ +#ifdef CONFIG_MMU +#ifdef CONFIG_ARCH_MAYBE_MKWRITE +pte_t arch_maybe_mkwrite (pte_t pte,struct vm_area_struct *vma); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +pmd_t arch_maybe_pmd_mkwrite (pmd_t pmd,struct vm_area_struct *vma); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#else /* !CONFIG_ARCH_MAYBE_MKWRITE */ +static inline pte_t arch_maybe_mkwrite (pte_t pte,struct vm_area_struct *vma) +{ +return pte; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t arch_maybe_pmd_mkwrite (pmd_t pmd,struct vm_area_struct *vma) +{ +return pmd; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#endif /* CONFIG_ARCH_MAYBE_MKWRITE */ +#endif /* CONFIG_MMU */ +/* * Architecture PAGE_KERNEL_* fallbacks *
mm/huge_memory.c +2 -0
@@ -464,6 +464,8 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) {if (likely (vma->vm_flags &VM_WRITE )) pmd =pmd_mkwrite (pmd); +else + pmd =arch_maybe_pmd_mkwrite (pmd, vma);return pmd; }
Signature · 1 line
2.21.0
PATCH v14 15/26 mm: Fixup places that call pte_mkwrite() directly
In reply to Yu-cheng YuA shadow stack page is made writable by pte_mkwrite_shstk(), which sets _PAGE_DIRTY_HW. There are a few places that call pte_mkwrite() directly and miss the maybe_mkwrite() fixup in the previous patch. Fix them with maybe_mkwrite():
- do_anonymous_page() and migrate_vma_insert_page() check VM_WRITE directly and call pte_mkwrite(), which is the same as maybe_mkwrite(). Change them to maybe_mkwrite().
- In do_numa_page(), if the numa entry 'was-writable', then pte_mkwrite() is called directly. Fix it by doing maybe_mkwrite().
- In change_pte_range(), pte_mkwrite() is called directly. Replace it with maybe_mkwrite().
--- mm/memory.c | 5 ++--- mm/migrate.c | 3 +-- mm/mprotect.c | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-)
mm/memory.c +2 -3
@@ -3563,8 +3563,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) entry =mk_pte (page, vma->vm_page_prot); entry =pte_sw_mkyoung (entry); -if (vma->vm_flags &VM_WRITE ) - entry =pte_mkwrite (pte_mkdirty (entry)); + entry =maybe_mkwrite (pte_mkdirty (entry), vma); vmf->pte =pte_offset_map_lock (vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl);
@@ -4218,7 +4217,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) pte =pte_modify (old_pte, vma->vm_page_prot); pte =pte_mkyoung (pte);if (was_writable) - pte =pte_mkwrite (pte); + pte =maybe_mkwrite (pte, vma);ptep_modify_prot_commit (vma, vmf->address, vmf->pte, old_pte, pte);update_mmu_cache (vma, vmf->address, vmf->pte);
mm/migrate.c +1 -2
@@ -2904,8 +2904,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, } }else { entry =mk_pte (page, vma->vm_page_prot); -if (vma->vm_flags &VM_WRITE ) - entry =pte_mkwrite (pte_mkdirty (entry)); + entry =maybe_mkwrite (pte_mkdirty (entry), vma); } ptep =pte_offset_map_lock (mm, pmdp, addr, &ptl);
mm/mprotect.c +1 -1
@@ -135,7 +135,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,if (dirty_accountable &&pte_dirty (ptent) && (pte_soft_dirty (ptent) || !(vma->vm_flags &VM_SOFTDIRTY ))) { - ptent =pte_mkwrite (ptent); + ptent =maybe_mkwrite (ptent, vma); }ptep_modify_prot_commit (vma, addr, pte, oldpte, ptent); pages++;
Signature · 1 line
2.21.0
PATCH v14 16/26 mm: Add guard pages around a shadow stack.
In reply to Yu-cheng YuINCSSP(Q/D) increments shadow stack pointer and 'pops and discards' the first and the last elements in the range, effectively touches those memory areas.
The maximum moving distance by INCSSPQ is 255 * 8 = 2040 bytes and 255 * 4 = 1020 bytes by INCSSPD. Both ranges are far from PAGE_SIZE. Thus, putting a gap page on both ends of a shadow stack prevents INCSSP, CALL, and RET from going beyond.
--- arch/x86/include/asm/processor.h | 10 ++++++++++ include/linux/mm.h | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-)
arch/x86/include/asm/processor.h +10 -0
@@ -840,6 +840,16 @@ static inline void spin_lock_prefetch(const void *x)#define STACK_TOP TASK_SIZE_LOW#define STACK_TOP_MAX TASK_SIZE_MAX +/* +* Shadow stack pointer is moved by CALL, JMP, and INCSSP(Q/D). INCSSPQ +* moves shadow stack pointer up to 255 * 8 = ~2 KB (~1KB for INCSSPD) and +* touches the first and the last element in the range, which triggers a +* page fault if the range is not in a shadow stack. Because of this, +* creating 4-KB guard pages around a shadow stack prevents these +* instructions from going beyond. +*/ +#define ARCH_SHADOW_STACK_GUARD_GAP PAGE_SIZE +#define INIT_THREAD { \ .addr_limit = KERNEL_DS, \ }
include/linux/mm.h +20 -4
@@ -2615,6 +2615,10 @@ extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);int __must_check write_one_page (struct page *page);void task_dirty_inc (struct task_struct *tsk); +#ifndef ARCH_SHADOW_STACK_GUARD_GAP +#define ARCH_SHADOW_STACK_GUARD_GAP 0 +#endif +extern unsigned long stack_guard_gap;/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ extern int expand_stack (struct vm_area_struct *vma,unsigned long address);
@@ -2647,9 +2651,15 @@ static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mstatic inline unsigned long vm_start_gap (struct vm_area_struct *vma) {unsigned long vm_start = vma->vm_start; +unsigned long gap =0 ; -if (vma->vm_flags &VM_GROWSDOWN ) { - vm_start -= stack_guard_gap; +if (vma->vm_flags &VM_GROWSDOWN ) + gap = stack_guard_gap; +else if (vma->vm_flags &VM_SHSTK ) + gap =ARCH_SHADOW_STACK_GUARD_GAP ; + +if (gap !=0 ) { + vm_start -= gap;if (vm_start > vma->vm_start) vm_start =0 ; }
@@ -2659,9 +2669,15 @@ static inline unsigned long vm_start_gap(struct vm_area_struct *vma)static inline unsigned long vm_end_gap (struct vm_area_struct *vma) {unsigned long vm_end = vma->vm_end; +unsigned long gap =0 ; + +if (vma->vm_flags &VM_GROWSUP ) + gap = stack_guard_gap; +else if (vma->vm_flags &VM_SHSTK ) + gap =ARCH_SHADOW_STACK_GUARD_GAP ; -if (vma->vm_flags &VM_GROWSUP ) { - vm_end += stack_guard_gap; +if (gap !=0 ) { + vm_end += gap;if (vm_end < vma->vm_end) vm_end = -PAGE_SIZE ; }
Signature · 1 line
2.21.0
PATCH v14 17/26 mm/mmap: Add shadow stack pages to memory accounting
In reply to Yu-cheng YuAccount shadow stack pages to stack memory.
--- arch/x86/mm/pgtable.c | 7 +++++++ include/linux/pgtable.h | 11 +++++++++++ mm/mmap.c | 5 +++++ 3 files changed, 23 insertions(+)
arch/x86/mm/pgtable.c +7 -0
@@ -893,3 +893,10 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)#endif /* CONFIG_X86_64 */ #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ + +#ifdef CONFIG_ARCH_HAS_SHADOW_STACK +bool arch_shadow_stack_mapping (vm_flags_t vm_flags) +{ +return (vm_flags &VM_SHSTK ); +} +#endif
include/linux/pgtable.h +11 -0
@@ -1380,6 +1380,17 @@ static inline pmd_t arch_maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma#endif /* CONFIG_ARCH_MAYBE_MKWRITE */ #endif /* CONFIG_MMU */ +#ifdef CONFIG_MMU +#ifdef CONFIG_ARCH_HAS_SHADOW_STACK +bool arch_shadow_stack_mapping (vm_flags_t vm_flags); +#else +static inline bool arch_shadow_stack_mapping (vm_flags_t vm_flags) +{ +return false; +} +#endif /* CONFIG_ARCH_HAS_SHADOW_STACK */ +#endif /* CONFIG_MMU */ +/* * Architecture PAGE_KERNEL_* fallbacks *
mm/mmap.c +5 -0
@@ -1682,6 +1682,9 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)if (file &&is_file_hugepages (file))return 0 ; +if (arch_shadow_stack_mapping (vm_flags)) +return 1 ; +return (vm_flags & (VM_NORESERVE |VM_SHARED |VM_WRITE )) ==VM_WRITE ; }
@@ -3352,6 +3355,8 @@ void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages) mm->stack_vm += npages; else if (is_data_mapping(flags)) mm->data_vm += npages; + else if (arch_shadow_stack_mapping(flags)) + mm->stack_vm += npages; }static vm_fault_t special_mapping_fault (struct vm_fault *vmf);
Signature · 1 line
2.21.0
PATCH v14 18/26 mm: Update can_follow_write_pte() for shadow stack
In reply to Yu-cheng YuCan_follow_write_pte() ensures a read-only page is COWed by checking the FOLL_COW flag, and uses pte_dirty() to validate the flag is still valid.
Like a writable data page, a shadow stack page is writable, and becomes read-only during copy-on-write, but it is always dirty. Thus, in the can_follow_write_pte() check, it belongs to the writable page case and should be excluded from the read-only page pte_dirty() check. Apply the same changes to can_follow_write_pmd().
--- mm/gup.c | 8 +++++--- mm/huge_memory.c | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-)
mm/gup.c +5 -3
@@ -384,10 +384,12 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,* FOLL_FORCE can write to even unwritable pte's, but only * after we've gone through a COW cycle and they are dirty. */ -static inline bool can_follow_write_pte (pte_t pte,unsigned int flags) +static inline bool can_follow_write_pte (pte_t pte,unsigned int flags, +struct vm_area_struct *vma) {return pte_write (pte) || - ((flags &FOLL_FORCE ) && (flags &FOLL_COW ) &&pte_dirty (pte)); + ((flags &FOLL_FORCE ) && (flags &FOLL_COW ) &&pte_dirty (pte) && + !arch_shadow_stack_mapping (vma->vm_flags)); } static struct page *follow_page_pte(struct vm_area_struct *vma,
@@ -430,7 +432,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, }if ((flags &FOLL_NUMA ) &&pte_protnone (pte)) goto no_page; -if ((flags &FOLL_WRITE ) && !can_follow_write_pte (pte, flags)) { +if ((flags &FOLL_WRITE ) && !can_follow_write_pte (pte, flags, vma)) {pte_unmap_unlock (ptep, ptl);return NULL ; }
mm/huge_memory.c +5 -3
@@ -1324,10 +1324,12 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)* FOLL_FORCE can write to even unwritable pmd's, but only * after we've gone through a COW cycle and they are dirty. */ -static inline bool can_follow_write_pmd (pmd_t pmd,unsigned int flags) +static inline bool can_follow_write_pmd (pmd_t pmd,unsigned int flags, +struct vm_area_struct *vma) {return pmd_write (pmd) || - ((flags &FOLL_FORCE ) && (flags &FOLL_COW ) &&pmd_dirty (pmd)); + ((flags &FOLL_FORCE ) && (flags &FOLL_COW ) &&pmd_dirty (pmd) && + !arch_shadow_stack_mapping (vma->vm_flags)); } struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
@@ -1340,7 +1342,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,assert_spin_locked (pmd_lockptr (mm, pmd)); -if (flags &FOLL_WRITE && !can_follow_write_pmd (*pmd, flags)) +if (flags &FOLL_WRITE && !can_follow_write_pmd (*pmd, flags, vma)) goto out;/* Avoid dumping huge zero page */
Signature · 1 line
2.21.0
PATCH v14 19/26 mm: Re-introduce vm_flags to do_mmap()
In reply to Yu-cheng YuThere was no more caller passing vm_flags to do_mmap(), and vm_flags was removed from the function's input by:
commit 45e55300f114 ("mm: remove unnecessary wrapper function do_mmap_pgoff()").
There is a new user now. Shadow stack allocation passes VM_SHSTK to do_mmap(). Re-introduce vm_flags to do_mmap(), but without the old wrapper do_mmap_pgoff(). Instead, make all callers of the wrapper pass a zero vm_flags to do_mmap().
--- fs/aio.c | 2 +- include/linux/mm.h | 3 ++- ipc/shm.c | 2 +- mm/mmap.c | 10 +++++----- mm/nommu.c | 4 ++-- mm/util.c | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-)
fs/aio.c +1 -1
@@ -527,7 +527,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) ctx->mmap_base =do_mmap (ctx->aio_ring_file,0 , ctx->mmap_size,PROT_READ |PROT_WRITE , -MAP_SHARED ,0 , &unused,NULL ); +MAP_SHARED ,0 ,0 , &unused,NULL );mmap_write_unlock (mm);if (IS_ERR ((void *)ctx->mmap_base)) { ctx->mmap_size =0 ;
include/linux/mm.h +2 -1
@@ -2560,7 +2560,8 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, struct list_head *uf);extern unsigned long do_mmap (struct file *file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flags, -unsigned long pgoff,unsigned long *populate,struct list_head *uf); +vm_flags_t vm_flags,unsigned long pgoff,unsigned long *populate, +struct list_head *uf);extern int __do_munmap (struct mm_struct *,unsigned long ,size_t ,struct list_head *uf,bool downgrade); extern int do_munmap(struct mm_struct *, unsigned long, size_t,
ipc/shm.c +1 -1
@@ -1556,7 +1556,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto invalid; } - addr =do_mmap (file, addr, size, prot, flags,0 , &populate,NULL ); + addr =do_mmap (file, addr, size, prot, flags,0 ,0 , &populate,NULL ); *raddr = addr; err =0 ;if (IS_ERR_VALUE (addr))
mm/mmap.c +5 -5
@@ -1365,11 +1365,11 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode,*/ unsigned long do_mmap (struct file *file,unsigned long addr,unsigned long len,unsigned long prot, -unsigned long flags,unsigned long pgoff, -unsigned long *populate,struct list_head *uf) +unsigned long flags,vm_flags_t vm_flags, +unsigned long pgoff,unsigned long *populate, +struct list_head *uf) {struct mm_struct *mm = current->mm; -vm_flags_t vm_flags;int pkey =0 ; *populate =0 ;
@@ -1431,7 +1431,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,* to. we assume access permissions have been handled by the open * of the memory object, so we don't do any here. */ - vm_flags =calc_vm_prot_bits (prot, pkey) |calc_vm_flag_bits (flags) | + vm_flags |=calc_vm_prot_bits (prot, pkey) |calc_vm_flag_bits (flags) | mm->def_flags |VM_MAYREAD |VM_MAYWRITE |VM_MAYEXEC ;if (flags &MAP_LOCKED )
@@ -3007,7 +3007,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, file =get_file (vma->vm_file); ret =do_mmap (vma->vm_file, start, size, - prot, flags, pgoff, &populate,NULL ); + prot, flags,0 , pgoff, &populate,NULL );fput (file); out:mmap_write_unlock (mm);
mm/nommu.c +2 -2
@@ -1078,6 +1078,7 @@ unsigned long do_mmap(struct file *file,unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf)
@@ -1085,7 +1086,6 @@ unsigned long do_mmap(struct file *file,struct vm_area_struct *vma;struct vm_region *region;struct rb_node *rb; -vm_flags_t vm_flags;unsigned long capabilities, result;int ret;
@@ -1104,7 +1104,7 @@ unsigned long do_mmap(struct file *file,/* we've determined that we can make the mapping, now translate what we * now know into VMA flags */ - vm_flags =determine_vm_flags (file, prot, flags, capabilities); + vm_flags |=determine_vm_flags (file, prot, flags, capabilities);/* we're going to need to record the mapping */ region =kmem_cache_zalloc (vm_region_jar,GFP_KERNEL );
mm/util.c +1 -1
@@ -503,7 +503,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,if (!ret) {if (mmap_write_lock_killable (mm))return -EINTR ; - ret =do_mmap (file, addr, len, prot, flag, pgoff, &populate, + ret =do_mmap (file, addr, len, prot, flag,0 , pgoff, &populate, &uf);mmap_write_unlock (mm);userfaultfd_unmap_complete (mm, &uf);
Signature · 1 line
2.21.0
PATCH v14 20/26 x86/cet/shstk: User-mode shadow stack support
In reply to Yu-cheng YuThis patch adds basic shadow stack enabling/disabling routines. A task's shadow stack is allocated from memory with VM_SHSTK flag and has a fixed size of min(RLIMIT_STACK, 4GB).
--- arch/x86/include/asm/cet.h | 28 ++++ arch/x86/include/asm/disabled-features.h | 8 +- arch/x86/include/asm/processor.h | 5 + arch/x86/kernel/Makefile | 2 + arch/x86/kernel/cet.c | 147 ++++++++++++++++++ arch/x86/kernel/cpu/common.c | 28 ++++ arch/x86/kernel/process.c | 1 + .../arch/x86/include/asm/disabled-features.h | 8 +- 8 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 arch/x86/include/asm/cet.h create mode 100644 arch/x86/kernel/cet.c
arch/x86/include/asm/cet.h new file +28 -0
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_CET_H +#define _ASM_X86_CET_H + +#ifndef __ASSEMBLY__ +#include <linux/types.h> + +struct task_struct ; +/* +* Per-thread CET status +*/ +struct cet_status { +unsigned long shstk_base; +unsigned long shstk_size; +}; + +#ifdef CONFIG_X86_CET +int cet_setup_shstk (void ); +void cet_disable_shstk (void ); +void cet_free_shstk (struct task_struct *p); +#else +static inline void cet_disable_shstk (void ) {} +static inline void cet_free_shstk (struct task_struct *p) {} +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_X86_CET_H */
arch/x86/include/asm/disabled-features.h +7 -1
@@ -56,6 +56,12 @@# define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31))#endif +#ifdef CONFIG_X86_SHADOW_STACK_USER +#define DISABLE_SHSTK 0 +#else +#define DISABLE_SHSTK (1<<(X86_FEATURE_SHSTK & 31)) +#endif +/* * Make sure to add features to the correct mask */
@@ -75,7 +81,7 @@#define DISABLED_MASK13 0#define DISABLED_MASK14 0#define DISABLED_MASK15 0 -#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP) +#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP|DISABLE_SHSTK)#define DISABLED_MASK17 0#define DISABLED_MASK18 0#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
arch/x86/include/asm/processor.h +5 -0
@@ -27,6 +27,7 @@ struct vm86;#include <asm/unwind_hints.h> #include <asm/vmxfeatures.h> #include <asm/vdso/processor.h> +#include <asm/cet.h> #include <linux/personality.h> #include <linux/cache.h>
@@ -542,6 +543,10 @@ struct thread_struct { unsigned int sig_on_uaccess_err:1; +#ifdef CONFIG_X86_CET +struct cet_status cet; +#endif +/* Floating point and extended processor state */ struct fpu fpu;/*
arch/x86/kernel/Makefile +2 -0
@@ -145,6 +145,8 @@ obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o +obj-$(CONFIG_X86_CET) += cet.o + ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y)
arch/x86/kernel/cet.c new file +147 -0
147 lines not on this page · raw message
arch/x86/kernel/cpu/common.c +28 -0
@@ -56,6 +56,7 @@#include <asm/microcode_intel.h> #include <asm/intel-family.h> #include <asm/cpu_device_id.h> +#include <asm/cet.h> #include <asm/uv/uv.h> #include "cpu.h"
@@ -509,6 +510,32 @@ static __init int setup_disable_pku(char *arg)__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +static __always_inline void setup_cet(struct cpuinfo_x86 *c) +{ +if (!cpu_feature_enabled (X86_FEATURE_SHSTK ) && + !cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +cr4_set_bits (X86_CR4_CET ); +} + +#ifdef CONFIG_X86_SHADOW_STACK_USER +static __init int setup_disable_shstk (char *s) +{ +/* require an exact match without trailing characters */ +if (s[0 ] !='\0' ) +return 0 ; + +if (!boot_cpu_has (X86_FEATURE_SHSTK )) +return 1 ; + +setup_clear_cpu_cap (X86_FEATURE_SHSTK ); +pr_info ("x86: 'no_user_shstk' specified, disabling user Shadow Stack\n" ); +return 1 ; +} +__setup ("no_user_shstk" , setup_disable_shstk); +#endif +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
@@ -1544,6 +1571,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)x86_init_rdrand (c);setup_pku (c); +setup_cet (c);/* * Clear/Set all flags overridden by options, need do it
arch/x86/kernel/process.c +1 -0
@@ -43,6 +43,7 @@#include <asm/io_bitmap.h> #include <asm/proto.h> #include <asm/frame.h> +#include <asm/cet.h> #include "process.h"
tools/arch/x86/include/asm/disabled-features.h +7 -1
@@ -56,6 +56,12 @@# define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31))#endif +#ifdef CONFIG_X86_SHADOW_STACK_USER +#define DISABLE_SHSTK 0 +#else +#define DISABLE_SHSTK (1<<(X86_FEATURE_SHSTK & 31)) +#endif +/* * Make sure to add features to the correct mask */
@@ -75,7 +81,7 @@#define DISABLED_MASK13 0#define DISABLED_MASK14 0#define DISABLED_MASK15 0 -#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP) +#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP|DISABLE_SHSTK)#define DISABLED_MASK17 0#define DISABLED_MASK18 0#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
Signature · 1 line
2.21.0
PATCH v14 21/26 x86/cet/shstk: Handle signals for shadow stack
In reply to Yu-cheng YuTo deliver a signal, create a shadow stack restore token and put a restore token and the signal restorer address on the shadow stack. For sigreturn, verify the token and restore the shadow stack pointer.
Introduce WRUSS, which is a kernel-mode instruction but writes directly to user shadow stack. It is used to construct the user signal stack as described above.
Introduce a signal context extension struct 'sc_ext', which is used to save shadow stack restore token address and WAIT_ENDBR status. WAIT_ENDBR will be introduced later in the Indirect Branch Tracking (IBT) series, but add that into sc_ext now to keep the struct stable in case the IBT series is applied later.
--- arch/x86/ia32/ia32_signal.c | 17 +++ arch/x86/include/asm/cet.h | 8 ++ arch/x86/include/asm/fpu/internal.h | 10 ++ arch/x86/include/asm/special_insns.h | 32 ++++++ arch/x86/include/uapi/asm/sigcontext.h | 9 ++ arch/x86/kernel/cet.c | 152 +++++++++++++++++++++++++ arch/x86/kernel/fpu/signal.c | 100 ++++++++++++++++ arch/x86/kernel/signal.c | 10 ++ 8 files changed, 338 insertions(+)
arch/x86/ia32/ia32_signal.c +17 -0
@@ -35,6 +35,7 @@#include <asm/sigframe.h> #include <asm/sighandling.h> #include <asm/smap.h> +#include <asm/cet.h> static inline void reload_segments (struct sigcontext_32 *sc) {
@@ -205,6 +206,7 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, void __user **fpstate) {unsigned long sp, fx_aligned, math_size; +void __user *restorer =NULL ;/* Default to using normal stack */ sp = regs->sp;
@@ -218,8 +220,23 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, ksig->ka.sa.sa_restorer) sp = (unsigned long ) ksig->ka.sa.sa_restorer; +if (ksig->ka.sa.sa_flags &SA_RESTORER ) { + restorer = ksig->ka.sa.sa_restorer; + }else if (current->mm->context.vdso) { +if (ksig->ka.sa.sa_flags &SA_SIGINFO ) + restorer = current->mm->context.vdso + + vdso_image_32.sym___kernel_rt_sigreturn; +else + restorer = current->mm->context.vdso + + vdso_image_32.sym___kernel_sigreturn; + } + sp =fpu__alloc_mathframe (sp,1 , &fx_aligned, &math_size); *fpstate = (struct _fpstate_32 __user *) sp; + +if (save_cet_to_sigframe (1 , *fpstate, (unsigned long )restorer)) +return (void __user *)-1L ; +if (copy_fpstate_to_sigframe (*fpstate, (void __user *)fx_aligned, math_size) <0 )return (void __user *)-1L ;
arch/x86/include/asm/cet.h +8 -0
@@ -6,6 +6,8 @@#include <linux/types.h> struct task_struct ; +struct sc_ext ; +/* * Per-thread CET status */
@@ -18,9 +20,15 @@ struct cet_status {int cet_setup_shstk (void );void cet_disable_shstk (void );void cet_free_shstk (struct task_struct *p); +int cet_verify_rstor_token (bool ia32,unsigned long ssp,unsigned long *new_ssp); +void cet_restore_signal (struct sc_ext *sc); +int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc);#else static inline void cet_disable_shstk (void ) {}static inline void cet_free_shstk (struct task_struct *p) {} +static inline void cet_restore_signal (struct sc_ext *sc) {return ; } +static inline int cet_setup_signal (bool ia32,unsigned long rstor, +struct sc_ext *sc) {return -EINVAL ; }#endif #endif /* __ASSEMBLY__ */
arch/x86/include/asm/fpu/internal.h +10 -0
@@ -442,6 +442,16 @@ static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate)__copy_kernel_to_fpregs (fpstate,-1 ); } +#ifdef CONFIG_X86_CET +extern int save_cet_to_sigframe (int ia32,void __user *fp, +unsigned long restorer); +#else +static inline int save_cet_to_sigframe (int ia32,void __user *fp, +unsigned long restorer) +{ +return 0 ; +} +#endif extern int copy_fpstate_to_sigframe (void __user *buf,void __user *fp,int size);/*
arch/x86/include/asm/special_insns.h +32 -0
@@ -232,6 +232,38 @@ static inline void clwb(volatile void *__p) : [pax] "a" (p)); } +#ifdef CONFIG_X86_CET +#if defined(CONFIG_IA32_EMULATION ) || defined(CONFIG_X86_X32 ) +static inline int write_user_shstk_32 (unsigned long addr,unsigned int val) +{ + asm_volatile_goto("1: wrussd %1, (%0)\n" + _ASM_EXTABLE(1b, %l[fail]) + :: "r" (addr), "r" (val) + :: fail); +return 0 ; +fail: +return -EPERM ; +} +#else +static inline int write_user_shstk_32 (unsigned long addr,unsigned int val) +{ +WARN_ONCE (1 ,"%s used but not supported.\n" , __func__); +return -EFAULT ; +} +#endif + +static inline int write_user_shstk_64 (unsigned long addr,unsigned long val) +{ + asm_volatile_goto("1: wrussq %1, (%0)\n" + _ASM_EXTABLE(1b, %l[fail]) + :: "r" (addr), "r" (val) + :: fail); +return 0 ; +fail: +return -EPERM ; +} +#endif /* CONFIG_X86_CET */ +#define nop () asm volatile ("nop")#endif /* __KERNEL__ */
arch/x86/include/uapi/asm/sigcontext.h +9 -0
@@ -196,6 +196,15 @@ struct _xstate {/* New processor state extensions go here: */ }; +/* +* Located at the end of sigcontext->fpstate, aligned to 8. +*/ +struct sc_ext { +unsigned long total_size; +unsigned long ssp; +unsigned long wait_endbr; +}; +/* * The 32-bit signal frame: */
arch/x86/kernel/cet.c +152 -0
167 lines not on this page · raw message
arch/x86/kernel/fpu/signal.c +100 -0
142 lines not on this page · raw message
arch/x86/kernel/signal.c +10 -0
@@ -46,6 +46,7 @@#include <asm/syscall.h> #include <asm/sigframe.h> #include <asm/signal.h> +#include <asm/cet.h> #ifdef CONFIG_X86_64 /*
@@ -239,6 +240,9 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,unsigned long buf_fx =0 ;int onsigstack =on_sig_stack (sp);int ret; +#ifdef CONFIG_X86_64 +void __user *restorer =NULL ; +#endif /* redzone */ if (IS_ENABLED (CONFIG_X86_64 ))
@@ -270,6 +274,12 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,if (onsigstack && !likely (on_sig_stack (sp)))return (void __user *)-1L ; +#ifdef CONFIG_X86_64 +if (ka->sa.sa_flags &SA_RESTORER ) + restorer = ka->sa.sa_restorer; + ret =save_cet_to_sigframe (0 , *fpstate, (unsigned long )restorer); +#endif +/* save i387 and extended state */ ret =copy_fpstate_to_sigframe (*fpstate, (void __user *)buf_fx, math_size);if (ret <0 )
Signature · 1 line
2.21.0
PATCH v14 22/26 binfmt_elf: Define GNU_PROPERTY_X86_FEATURE_1_AND properties
In reply to Yu-cheng YuAn ELF file's .note.gnu.property indicates architecture features of the file.. Introduce feature definitions for Shadow Stack and Indirect Branch Tracking.
--- include/uapi/linux/elf.h | 9 +++++++++ 1 file changed, 9 insertions(+)
include/uapi/linux/elf.h +9 -0
@@ -454,4 +454,13 @@ typedef struct elf64_note {/* Bits for GNU_PROPERTY_AARCH64_FEATURE_1_BTI */ #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) +/* .note.gnu.property types for x86: */ +#define GNU_PROPERTY_X86_FEATURE_1_AND 0xc0000002 + +/* Bits for GNU_PROPERTY_X86_FEATURE_1_AND */ +#define GNU_PROPERTY_X86_FEATURE_1_IBT 0x00000001 +#define GNU_PROPERTY_X86_FEATURE_1_SHSTK 0x00000002 +#define GNU_PROPERTY_X86_FEATURE_1_INVAL ~(GNU_PROPERTY_X86_FEATURE_1_IBT | \ + GNU_PROPERTY_X86_FEATURE_1_SHSTK) +#endif /* _UAPI_LINUX_ELF_H */
Signature · 1 line
2.21.0
PATCH v14 23/26 ELF: Introduce arch_setup_elf_property()
In reply to Yu-cheng YuAn ELF file's .note.gnu.property indicates arch features supported by the file. These features are extracted by arch_parse_elf_property() and stored in 'arch_elf_state'. Introduce arch_setup_elf_property() for enabling such features. The first use-case of this function is shadow stack.
ARM64 is the other arch that has ARCH_USER_GNU_PROPERTY and arch_parse_elf_ property(). Add arch_setup_elf_property() for it.
--- arch/arm64/include/asm/elf.h | 5 +++++ arch/x86/Kconfig | 2 ++ arch/x86/include/asm/elf.h | 13 +++++++++++++ arch/x86/kernel/process_64.c | 32 ++++++++++++++++++++++++++++++++ fs/binfmt_elf.c | 4 ++++ include/linux/elf.h | 6 ++++++ 6 files changed, 62 insertions(+)
arch/arm64/include/asm/elf.h +5 -0
@@ -281,6 +281,11 @@ static inline int arch_parse_elf_property(u32 type, const void *data,return 0 ; } +static inline int arch_setup_elf_property (struct arch_elf_state *arch) +{ +return 0 ; +} +static inline int arch_elf_pt_proc (void *ehdr,void *phdr,struct file *f,bool is_interp,struct arch_elf_state *state)
arch/x86/Kconfig +2 -0
@@ -1950,6 +1950,8 @@ config X86_SHADOW_STACK_USER select X86_CET select ARCH_MAYBE_MKWRITE select ARCH_HAS_SHADOW_STACK + select ARCH_USE_GNU_PROPERTY + select ARCH_BINFMT_ELF_STATE help Shadow Stacks provides protection against program stack corruption. It's a hardware feature. This only matters
arch/x86/include/asm/elf.h +13 -0
@@ -385,6 +385,19 @@ extern int compat_arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp);#define compat_arch_setup_additional_pages compat_arch_setup_additional_pages +#ifdef CONFIG_ARCH_BINFMT_ELF_STATE +struct arch_elf_state { +unsigned int gnu_property; +}; + +#define INIT_ARCH_ELF_STATE { \ + .gnu_property = 0, \ +} + +#define arch_elf_pt_proc (ehdr, phdr, elf, interp, state) (0) +#define arch_check_elf (ehdr, interp, interp_ehdr, state) (0) +#endif +/* Do not change the values. See get_align_mask() */ enum align_flags {ALIGN_VA_32 =BIT (0 ),
arch/x86/kernel/process_64.c +32 -0
@@ -837,3 +837,35 @@ unsigned long KSTK_ESP(struct task_struct *task) {return task_pt_regs (task)->sp; } + +#ifdef CONFIG_ARCH_USE_GNU_PROPERTY +int arch_parse_elf_property (u32 type,const void *data,size_t datasz, +bool compat,struct arch_elf_state *state) +{ +if (type !=GNU_PROPERTY_X86_FEATURE_1_AND ) +return 0 ; + +if (datasz !=sizeof (unsigned int )) +return -ENOEXEC ; + + state->gnu_property = *(unsigned int *)data; +return 0 ; +} + +int arch_setup_elf_property (struct arch_elf_state *state) +{ +int r =0 ; + +if (!IS_ENABLED (CONFIG_X86_CET )) +return r; + +memset (¤t->thread.cet,0 ,sizeof (struct cet_status )); + +if (static_cpu_has (X86_FEATURE_SHSTK )) { +if (state->gnu_property &GNU_PROPERTY_X86_FEATURE_1_SHSTK ) + r =cet_setup_shstk (); + } + +return r; +} +#endif
fs/binfmt_elf.c +4 -0
@@ -1217,6 +1217,10 @@ static int load_elf_binary(struct linux_binprm *bprm)set_binfmt (&elf_format); + retval =arch_setup_elf_property (&arch_state); +if (retval <0 ) + goto out; +#ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES retval =arch_setup_additional_pages (bprm, !!interpreter);if (retval <0 )
include/linux/elf.h +6 -0
@@ -82,9 +82,15 @@ static inline int arch_parse_elf_property(u32 type, const void *data, {return 0 ; } + +static inline int arch_setup_elf_property (struct arch_elf_state *arch) +{ +return 0 ; +}#else extern int arch_parse_elf_property (u32 type,const void *data,size_t datasz,bool compat,struct arch_elf_state *arch); +extern int arch_setup_elf_property (struct arch_elf_state *arch);#endif #ifdef CONFIG_ARCH_HAVE_ELF_PROT
Signature · 1 line
2.21.0
PATCH v14 24/26 x86/cet/shstk: Handle thread shadow stack
In reply to Yu-cheng YuThe kernel allocates (and frees on thread exit) a new shadow stack for a pthread child.
It is possible for the kernel to complete the clone syscall and set the child's shadow stack pointer to NULL and let the child thread allocate a shadow stack for itself. There are two issues in this approach: It is not compatible with existing code that does inline syscall and it cannot handle signals before the child can successfully allocate a shadow stack.
A 64-bit shadow stack has a size of min(RLIMIT_STACK, 4 GB). A compat-mode thread shadow stack has a size of 1/4 min(RLIMIT_STACK, 4 GB). This allows more threads to run in a 32-bit address space.
--- arch/x86/include/asm/cet.h | 3 ++ arch/x86/include/asm/mmu_context.h | 3 ++ arch/x86/kernel/cet.c | 44 ++++++++++++++++++++++++++++++ arch/x86/kernel/process.c | 7 +++++ 4 files changed, 57 insertions(+)
arch/x86/include/asm/cet.h +3 -0
@@ -18,12 +18,15 @@ struct cet_status {#ifdef CONFIG_X86_CET int cet_setup_shstk (void ); +int cet_setup_thread_shstk (struct task_struct *p,unsigned long clone_flags);void cet_disable_shstk (void );void cet_free_shstk (struct task_struct *p);int cet_verify_rstor_token (bool ia32,unsigned long ssp,unsigned long *new_ssp);void cet_restore_signal (struct sc_ext *sc);int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc);#else +static inline int cet_setup_thread_shstk (struct task_struct *p, +unsigned long clone_flags) {return 0 ; }static inline void cet_disable_shstk (void ) {}static inline void cet_free_shstk (struct task_struct *p) {}static inline void cet_restore_signal (struct sc_ext *sc) {return ; }
arch/x86/include/asm/mmu_context.h +3 -0
@@ -11,6 +11,7 @@#include <asm/tlbflush.h> #include <asm/paravirt.h> +#include <asm/cet.h> #include <asm/debugreg.h> extern atomic64_t last_mm_ctx_id;
@@ -142,6 +143,8 @@ do { \#else #define deactivate_mm (tsk, mm) \ do { \ + if (!tsk->vfork_done) \ + cet_free_shstk(tsk); \ load_gs_index(0); \ loadsegment(fs, 0); \ } while (0)
arch/x86/kernel/cet.c +44 -0
50 lines not on this page · raw message
arch/x86/kernel/process.c +7 -0
@@ -110,6 +110,7 @@ void exit_thread(struct task_struct *tsk)free_vm86 (t); +cet_free_shstk (tsk);fpu__drop (fpu); }
@@ -182,6 +183,12 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg,if (clone_flags &CLONE_SETTLS ) ret =set_new_tls (p, tls); +#ifdef CONFIG_X86_64 +/* Allocate a new shadow stack for pthread */ +if (!ret) + ret =cet_setup_thread_shstk (p, clone_flags); +#endif +if (!ret &&unlikely (test_tsk_thread_flag (current,TIF_IO_BITMAP )))io_bitmap_share (p);
Signature · 1 line
2.21.0
PATCH v14 25/26 x86/cet/shstk: Add arch_prctl functions for shadow stack
In reply to Yu-cheng Yuarch_prctl(ARCH_X86_CET_STATUS, u64 *args)
Get CET feature status.
The parameter 'args' is a pointer to a user buffer. The kernel returns the following information:
*args = shadow stack/IBT status
*(args + 1) = shadow stack base address
*(args + 2) = shadow stack size
arch_prctl(ARCH_X86_CET_DISABLE, unsigned int features)
Disable CET features specified in 'features'. Return -EPERM if CET is locked.
arch_prctl(ARCH_X86_CET_LOCK)
Lock in CET features.
Also change do_arch_prctl_common()'s parameter 'cpuid_enabled' to 'arg2', as it is now also passed to prctl_cet().
--- arch/x86/include/asm/cet.h | 3 ++ arch/x86/include/uapi/asm/prctl.h | 4 ++ arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/cet_prctl.c | 68 +++++++++++++++++++++++++ arch/x86/kernel/process.c | 6 +-- tools/arch/x86/include/uapi/asm/prctl.h | 4 ++ 6 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 arch/x86/kernel/cet_prctl.c
arch/x86/include/asm/cet.h +3 -0
@@ -14,9 +14,11 @@ struct sc_ext;struct cet_status {unsigned long shstk_base;unsigned long shstk_size; +unsigned int locked:1 ; };#ifdef CONFIG_X86_CET +int prctl_cet (int option,u64 arg2);int cet_setup_shstk (void );int cet_setup_thread_shstk (struct task_struct *p,unsigned long clone_flags);void cet_disable_shstk (void );
@@ -25,6 +27,7 @@ int cet_verify_rstor_token(bool ia32, unsigned long ssp, unsigned long *new_ssp)void cet_restore_signal (struct sc_ext *sc);int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc);#else +static inline int prctl_cet (int option,u64 arg2) {return -EINVAL ; }static inline int cet_setup_thread_shstk (struct task_struct *p,unsigned long clone_flags) {return 0 ; }static inline void cet_disable_shstk (void ) {}
arch/x86/include/uapi/asm/prctl.h +4 -0
@@ -14,4 +14,8 @@#define ARCH_MAP_VDSO_32 0x2002#define ARCH_MAP_VDSO_64 0x2003 +#define ARCH_X86_CET_STATUS 0x3001 +#define ARCH_X86_CET_DISABLE 0x3002 +#define ARCH_X86_CET_LOCK 0x3003 +#endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/Makefile +1 -1
@@ -145,7 +145,7 @@ obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o -obj-$(CONFIG_X86_CET) += cet.o +obj-$(CONFIG_X86_CET) += cet.o cet_prctl.o ### # 64 bit specific files
arch/x86/kernel/cet_prctl.c new file +68 -0
68 lines not on this page · raw message
arch/x86/kernel/process.c +3 -3
@@ -977,14 +977,14 @@ unsigned long get_wchan(struct task_struct *p) }long do_arch_prctl_common (struct task_struct *task,int option, -unsigned long cpuid_enabled) +unsigned long arg2) {switch (option) {case ARCH_GET_CPUID :return get_cpuid_mode ();case ARCH_SET_CPUID : -return set_cpuid_mode (task, cpuid_enabled); +return set_cpuid_mode (task, arg2); } -return -EINVAL ; +return prctl_cet (option, arg2); }
tools/arch/x86/include/uapi/asm/prctl.h +4 -0
@@ -14,4 +14,8 @@#define ARCH_MAP_VDSO_32 0x2002#define ARCH_MAP_VDSO_64 0x2003 +#define ARCH_X86_CET_STATUS 0x3001 +#define ARCH_X86_CET_DISABLE 0x3002 +#define ARCH_X86_CET_LOCK 0x3003 +#endif /* _ASM_X86_PRCTL_H */
Signature · 1 line
2.21.0
PATCH v14 26/26 mm: Introduce PROT_SHSTK for shadow stack
In reply to Yu-cheng YuThere are three possible options to create a shadow stack allocation API:
an arch_prctl, a new syscall, or adding PROT_SHSTK to mmap()/mprotect(). Each has its advantages and compromises.
An arch_prctl() is the least intrusive. However, the existing x86 arch_prctl() takes only two parameters. Multiple parameters must be passed in a memory buffer. There is a proposal to pass more parameters in registers [1], but no active discussion on that.
A new syscall minimizes compatibility issues and offers an extensible frame work to other architectures, but this will likely result in some overlap of mmap()/mprotect().
The introduction of PROT_SHSTK to mmap()/mprotect() takes advantage of existing APIs. The x86-specific PROT_SHSTK is translated to VM_SHSTK and a shadow stack mapping is created without reinventing the wheel. There are potential pitfalls though. The most obvious one would be using this as a bypass to shadow stack protection. However, the attacker would have to get to the syscall first.
Since arch_calc_vm_prot_bits() is modified, I have moved arch_vm_get_page _prot() and arch_calc_vm_prot_bits() to x86/include/asm/mman.h. This will be more consistent with other architectures.
[1] https://lore.kernel.org/lkml/[email protected]/
--- arch/x86/include/asm/mman.h | 83 ++++++++++++++++++++++++++++++++ arch/x86/include/uapi/asm/mman.h | 28 ++--------- include/linux/mm.h | 1 + mm/mmap.c | 8 ++- 4 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 arch/x86/include/asm/mman.h
arch/x86/include/asm/mman.h new file +83 -0
@@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_MMAN_H +#define _ASM_X86_MMAN_H + +#include <linux/mm.h> +#include <uapi/asm/mman.h> + +#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS +/* +* Take the 4 protection key bits out of the vma->vm_flags +* value and turn them in to the bits that we can put in +* to a pte. +* +* Only override these if Protection Keys are available +* (which is only on 64-bit). +*/ +#define arch_vm_get_page_prot (vm_flags) __pgprot( \ + ((vm_flags) & VM_PKEY_BIT0 ? _PAGE_PKEY_BIT0 : 0) | \ + ((vm_flags) & VM_PKEY_BIT1 ? _PAGE_PKEY_BIT1 : 0) | \ + ((vm_flags) & VM_PKEY_BIT2 ? _PAGE_PKEY_BIT2 : 0) | \ + ((vm_flags) & VM_PKEY_BIT3 ? _PAGE_PKEY_BIT3 : 0)) + +#define pkey_vm_prot_bits (prot, key) ( \ + ((key) & 0x1 ? VM_PKEY_BIT0 : 0) | \ + ((key) & 0x2 ? VM_PKEY_BIT1 : 0) | \ + ((key) & 0x4 ? VM_PKEY_BIT2 : 0) | \ + ((key) & 0x8 ? VM_PKEY_BIT3 : 0)) +#else +#define pkey_vm_prot_bits (prot, key) (0) +#endif + +static inline unsigned long arch_calc_vm_prot_bits (unsigned long prot, +unsigned long pkey) +{ +unsigned long vm_prot_bits =pkey_vm_prot_bits (prot, pkey); + +if (!(prot &PROT_WRITE ) && (prot &PROT_SHSTK )) + vm_prot_bits |=VM_SHSTK ; + +return vm_prot_bits; +} +#define arch_calc_vm_prot_bits (prot, pkey) arch_calc_vm_prot_bits(prot, pkey) + +#ifdef CONFIG_X86_SHADOW_STACK_USER +static inline bool arch_validate_prot (unsigned long prot,unsigned long addr) +{ +unsigned long valid =PROT_READ |PROT_WRITE |PROT_EXEC |PROT_SEM ; + +if (prot & ~(valid |PROT_SHSTK )) +return false; + +if (prot &PROT_SHSTK ) { +struct vm_area_struct *vma; + +if (!current->thread.cet.shstk_size) +return false; + +/* +* A shadow stack mapping is indirectly writable by only +* the CALL and WRUSS instructions, but not other write +* instructions). PROT_SHSTK and PROT_WRITE are mutually +* exclusive. +*/ +if (prot &PROT_WRITE ) +return false; + + vma =find_vma (current->mm, addr); +if (!vma) +return false; + +/* +* Shadow stack cannot be backed by a file or shared. +*/ +if (vma->vm_file || (vma->vm_flags &VM_SHARED )) +return false; + } + +return true; +} +#define arch_validate_prot arch_validate_prot +#endif + +#endif /* _ASM_X86_MMAN_H */
arch/x86/include/uapi/asm/mman.h +4 -24
@@ -1,31 +1,11 @@/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _ASM_X86_MMAN_H -#define _ASM_X86_MMAN_H +#ifndef _UAPI_ASM_X86_MMAN_H +#define _UAPI_ASM_X86_MMAN_H#define MAP_32BIT 0x40/* only give out 32bit addresses */ -#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS -/* -* Take the 4 protection key bits out of the vma->vm_flags -* value and turn them in to the bits that we can put in -* to a pte. -* -* Only override these if Protection Keys are available -* (which is only on 64-bit). -*/ -#define arch_vm_get_page_prot (vm_flags) __pgprot( \ - ((vm_flags) & VM_PKEY_BIT0 ? _PAGE_PKEY_BIT0 : 0) | \ - ((vm_flags) & VM_PKEY_BIT1 ? _PAGE_PKEY_BIT1 : 0) | \ - ((vm_flags) & VM_PKEY_BIT2 ? _PAGE_PKEY_BIT2 : 0) | \ - ((vm_flags) & VM_PKEY_BIT3 ? _PAGE_PKEY_BIT3 : 0)) - -#define arch_calc_vm_prot_bits (prot, key) ( \ - ((key) & 0x1 ? VM_PKEY_BIT0 : 0) | \ - ((key) & 0x2 ? VM_PKEY_BIT1 : 0) | \ - ((key) & 0x4 ? VM_PKEY_BIT2 : 0) | \ - ((key) & 0x8 ? VM_PKEY_BIT3 : 0)) -#endif +#define PROT_SHSTK 0x10/* shadow stack pages */ #include <asm-generic/mman.h> -#endif /* _ASM_X86_MMAN_H */ +#endif /* _UAPI_ASM_X86_MMAN_H */
include/linux/mm.h +1 -0
@@ -334,6 +334,7 @@ extern unsigned int kobjsize(const void *objp);#if defined(CONFIG_X86 )# define VM_PAT VM_ARCH_1/* PAT reserves whole VMA at once (x86) */ +# define VM_ARCH_CLEAR VM_SHSTK#elif defined(CONFIG_PPC )# define VM_SAO VM_ARCH_1/* Strong Access Ordering (powerpc) */ #elif defined(CONFIG_PARISC )
mm/mmap.c +7 -1
@@ -1445,6 +1445,12 @@ unsigned long do_mmap(struct file *file, unsigned long addr,struct inode *inode =file_inode (file);unsigned long flags_mask; +/* +* Call stack cannot be backed by a file. +*/ +if (vm_flags &VM_SHSTK ) +return -EINVAL ; +if (!file_mmap_ok (file, inode, pgoff, len))return -EOVERFLOW ;
@@ -1509,7 +1515,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, }else {switch (flags &MAP_TYPE ) {case MAP_SHARED : -if (vm_flags & (VM_GROWSDOWN |VM_GROWSUP )) +if (vm_flags & (VM_GROWSDOWN |VM_GROWSUP |VM_SHSTK ))return -EINVAL ;/* * Ignore pgoff.
Signature · 1 line
2.21.0