scs: switch to vmapped shadow stacks
v3 cover scs: switch to vmapped shadow stacks
As discussed a few months ago [1][2], virtually mapped shadow call stacks are better for safety and robustness. This series dusts off the VMAP option from the original SCS patch series and switches the kernel to use virtually mapped shadow stacks unconditionally when SCS is enabled.
[1] https://lore.kernel.org/lkml/20200515172355.GD23334@willie-the-truck/
[2] https://lore.kernel.org/lkml/[email protected]/
Changes in v3:
- Split the actual allocation to __scs_alloc().
- Moved SDEI SCS initialization to init_sdei_scs().
Changes in v2:
- Added SCS_ORDER and used it to define SCS_SIZE, switched vmalloc() to use SCS_SIZE and removed the alignment.
- Moved the kasan_unpoison_vmalloc() to scs_alloc() when using a cached shadow stack instead of calling it in scs_free().
- Added a comment to scs_free().
- Moved arm64 IRQ and SDEI shadow stack initialization to irq/sdei.c, and removed the now unneeded scs.c.
Sami Tolvanen (2):
scs: switch to vmapped shadow stacks
arm64: scs: use vmapped IRQ and SDEI shadow stacks
arch/arm64/kernel/Makefile | 1 - arch/arm64/kernel/entry.S | 6 ++-- arch/arm64/kernel/irq.c | 19 ++++++++++ arch/arm64/kernel/scs.c | 16 --------- arch/arm64/kernel/sdei.c | 70 +++++++++++++++++++++++++++++++++++++ include/linux/scs.h | 16 ++++----- kernel/scs.c | 71 ++++++++++++++++++++++++++++++++------ 7 files changed, 158 insertions(+), 41 deletions(-) delete mode 100644 arch/arm64/kernel/scs.c
base-commit: b65054597872ce3aefbc6a666385eabdf9e288da
Signature · 1 line
2.29.2.454.gaff20da3a2-goog
PATCH v3 1/2 scs: switch to vmapped shadow stacks
The kernel currently uses kmem_cache to allocate shadow call stacks, which means an overflows may not be immediately detected and can potentially result in another task's shadow stack to be overwritten.
This change switches SCS to use virtually mapped shadow stacks for tasks, which increases shadow stack size to a full page and provides more robust overflow detection, similarly to VMAP_STACK.
--- include/linux/scs.h | 12 ++++---- kernel/scs.c | 71 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 17 deletions(-)
include/linux/scs.h +6 -6
@@ -15,12 +15,8 @@#ifdef CONFIG_SHADOW_CALL_STACK -/* -* In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit -* architecture) provided ~40% safety margin on stack usage while keeping -* memory allocation overhead reasonable. -*/ -#define SCS_SIZE SZ_1K +#define SCS_ORDER 0 +#define SCS_SIZE (PAGE_SIZE << SCS_ORDER)#define GFP_SCS (GFP_KERNEL | __GFP_ZERO)/* An illegal pointer value to mark the end of the shadow stack. */
@@ -33,6 +29,8 @@#define task_scs (tsk) (task_thread_info(tsk)->scs_base)#define task_scs_sp (tsk) (task_thread_info(tsk)->scs_sp) +void *scs_alloc (int node); +void scs_free (void *s);void scs_init (void );int scs_prepare (struct task_struct *tsk,int node);void scs_release (struct task_struct *tsk);
@@ -61,6 +59,8 @@ static inline bool task_scs_end_corrupted(struct task_struct *tsk)#else /* CONFIG_SHADOW_CALL_STACK */ +static inline void *scs_alloc (int node) {return NULL ; } +static inline void scs_free (void *s) {}static inline void scs_init (void ) {}static inline void scs_task_reset (struct task_struct *tsk) {}static inline int scs_prepare (struct task_struct *tsk,int node) {return 0 ; }
kernel/scs.c +60 -11
107 lines not on this page · raw message
Signature · 1 line
2.29.2.454.gaff20da3a2-goog
PATCH v3 2/2 arm64: scs: use vmapped IRQ and SDEI shadow stacks
In reply to Sami TolvanenUse scs_alloc() to allocate also IRQ and SDEI shadow stacks instead of using statically allocated stacks.
--- arch/arm64/kernel/Makefile | 1 - arch/arm64/kernel/entry.S | 6 ++-- arch/arm64/kernel/irq.c | 19 +++++++++++ arch/arm64/kernel/scs.c | 16 --------- arch/arm64/kernel/sdei.c | 70 ++++++++++++++++++++++++++++++++++++++ include/linux/scs.h | 4 --- 6 files changed, 92 insertions(+), 24 deletions(-) delete mode 100644 arch/arm64/kernel/scs.c
arch/arm64/kernel/Makefile +0 -1
@@ -58,7 +58,6 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o -obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso/ probes/
arch/arm64/kernel/entry.S +3 -3
@@ -441,7 +441,7 @@ SYM_CODE_END(__swpan_exit_el0) #ifdef CONFIG_SHADOW_CALL_STACK /* also switch to the irq shadow stack */ - adr_this_cpu scs_sp, irq_shadow_call_stack, x26 + ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26 #endif 9998:
@@ -1097,9 +1097,9 @@ SYM_CODE_START(__sdei_asm_handler) #ifdef CONFIG_SHADOW_CALL_STACK /* Use a separate shadow call stack for normal and critical events */ cbnz w4, 3f - adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6 + ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 b 4f -3: adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6 +3: ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 4: #endif
arch/arm64/kernel/irq.c +19 -0
@@ -17,6 +17,7 @@#include <linux/init.h> #include <linux/irqchip.h> #include <linux/kprobes.h> +#include <linux/scs.h> #include <linux/seq_file.h> #include <linux/vmalloc.h> #include <asm/daifflags.h>
@@ -27,6 +28,22 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); + +DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); + +#ifdef CONFIG_SHADOW_CALL_STACK +DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); +#endif + +static void init_irq_scs (void ) +{ +int cpu; + +for_each_possible_cpu (cpu) +per_cpu (irq_shadow_call_stack_ptr, cpu) = +scs_alloc (cpu_to_node (cpu)); +} +#ifdef CONFIG_VMAP_STACK static void init_irq_stacks (void ) {
@@ -54,6 +71,8 @@ static void init_irq_stacks(void)void __init init_IRQ (void ) {init_irq_stacks (); +if (IS_ENABLED (CONFIG_SHADOW_CALL_STACK )) +init_irq_scs ();irqchip_init ();if (!handle_arch_irq)panic ("No interrupt controller found." );
arch/arm64/kernel/scs.c deleted +0 -16
@@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* -* Shadow Call Stack support. -* -* Copyright (C) 2019 Google LLC -*/ - -#include <linux/percpu.h> -#include <linux/scs.h> - -DEFINE_SCS (irq_shadow_call_stack); - -#ifdef CONFIG_ARM_SDE_INTERFACE -DEFINE_SCS (sdei_shadow_call_stack_normal); -DEFINE_SCS (sdei_shadow_call_stack_critical); -#endif
arch/arm64/kernel/sdei.c +70 -0
94 lines not on this page · raw message
include/linux/scs.h +0 -4
@@ -22,10 +22,6 @@/* An illegal pointer value to mark the end of the shadow stack. */ #define SCS_END_MAGIC (0x5f6UL + POISON_POINTER_DELTA) -/* Allocate a static per-CPU shadow stack */ -#define DEFINE_SCS (name) \ - DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \ -#define task_scs (tsk) (task_thread_info(tsk)->scs_base)#define task_scs_sp (tsk) (task_thread_info(tsk)->scs_sp)
Signature · 1 line
2.29.2.454.gaff20da3a2-goog
Quoted 9 lines · Sami Tolvanen
As discussed a few months ago [1][2], virtually mapped shadow call stacks are better for safety and robustness. This series dusts off the VMAP option from the original SCS patch series and switches the kernel to use virtually mapped shadow stacks unconditionally when SCS is enabled.
[1] https://lore.kernel.org/lkml/20200515172355.GD23334@willie-the-truck/
[2] https://lore.kernel.org/lkml/[email protected]/[...]
Applied to arm64 (for-next/scs), thanks!
[1/2] scs: switch to vmapped shadow stacks
https://git.kernel.org/arm64/c/a2abe7cbd8fe [2/2] arm64: scs: use vmapped IRQ and SDEI shadow stacks https://git.kernel.org/arm64/c/ac20ffbb0279
I also threw a patch on top implementing the suggestion I made on v2, so please take a look if you get a chance.
Cheers,
Signature · 5 lines
Will https://fixes.arm64.dev https://next.arm64.dev https://will.arm64.dev
Quoted 14 lines · Will Deacon
Quoted 9 lines · Sami Tolvanen
As discussed a few months ago [1][2], virtually mapped shadow call stacks are better for safety and robustness. This series dusts off the VMAP option from the original SCS patch series and switches the kernel to use virtually mapped shadow stacks unconditionally when SCS is enabled.
[1] https://lore.kernel.org/lkml/20200515172355.GD23334@willie-the-truck/
[2] https://lore.kernel.org/lkml/[email protected]/[...]
Applied to arm64 (for-next/scs), thanks!
Quoted 7 lines
[1/2] scs: switch to vmapped shadow stacks
https://git.kernel.org/arm64/c/a2abe7cbd8fe [2/2] arm64: scs: use vmapped IRQ and SDEI shadow stacks https://git.kernel.org/arm64/c/ac20ffbb0279I also threw a patch on top implementing the suggestion I made on v2, so please take a look if you get a chance.
Looks good to me, thanks for cleaning that up!
Sami