add support for Clang's Shadow Call Stack
Subsystems (16)
PATCH v9 2/12 scs: add accounting
In reply to Sami TolvanenThis change adds accounting for the memory allocated for shadow stacks.
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 20 ++++++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid,sum_zone_node_page_state (nid,NR_KERNEL_SCS_BYTES ) /1024 , +#endif nid,K (sum_zone_node_page_state (nid,NR_PAGETABLE )), nid,K (node_page_state (pgdat,NR_UNSTABLE_NFS )), nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)show_val_kb (m,"SUnreclaim: " , sunreclaim);seq_printf (m,"KernelStack: %8lu kB\n" ,global_zone_page_state (NR_KERNEL_STACK_KB )); +#ifdef CONFIG_SHADOW_CALL_STACK +seq_printf (m,"ShadowCallStack:%8lu kB\n" , +global_zone_page_state (NR_KERNEL_SCS_BYTES ) /1024 ); +#endif show_val_kb (m,"PageTables: " ,global_zone_page_state (NR_PAGETABLE ));
include/linux/mmzone.h +3 -0
@@ -200,6 +200,9 @@ enum zone_stat_item {NR_MLOCK ,/* mlock()ed pages found and moved off LRU */ NR_PAGETABLE ,/* used for pagetables */ NR_KERNEL_STACK_KB ,/* measured in KiB */ +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +NR_KERNEL_SCS_BYTES ,/* measured in bytes */ +#endif/* Second 128 byte cacheline */ NR_BOUNCE , #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
@@ -12,6 +12,7 @@#include <linux/scs.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/vmstat.h> #include <asm/scs.h> static inline void *__scs_base (struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)vfree_atomic (s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return vmalloc_to_page (__scs_base (tsk)); +} +static int scs_cleanup (unsigned int cpu) {int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)kmem_cache_free (scs_cache, s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return virt_to_page (__scs_base (tsk)); +} +void __init scs_init (void ) { scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)task_set_scs (tsk,__scs_base (tsk)); } +static void scs_account (struct task_struct *tsk,int account) +{ +mod_zone_page_state (page_zone (__scs_page (tsk)),NR_KERNEL_SCS_BYTES , + account *SCS_SIZE ); +} +int scs_prepare (struct task_struct *tsk,int node) {void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)return -ENOMEM ;task_set_scs (tsk, s); +scs_account (tsk,1 ); +return 0 ; }
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)WARN_ON (scs_corrupted (tsk)); +scs_account (tsk,-1 );task_set_scs (tsk,NULL );scs_free (s); }
mm/page_alloc.c +6 -0
@@ -5340,6 +5340,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " managed:%lukB" " mlocked:%lukB" " kernel_stack:%lukB" +#ifdef CONFIG_SHADOW_CALL_STACK + " shadow_call_stack:%lukB" +#endif" pagetables:%lukB" " bounce:%lukB" " free_pcp:%lukB"
@@ -5362,6 +5365,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)K (zone_managed_pages (zone)),K (zone_page_state (zone,NR_MLOCK )),zone_page_state (zone,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK +zone_page_state (zone,NR_KERNEL_SCS_BYTES ) /1024 , +#endifK (zone_page_state (zone,NR_PAGETABLE )),K (zone_page_state (zone,NR_BOUNCE )), K(free_pcp),
mm/vmstat.c +3 -0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {"nr_mlock" ,"nr_page_table_pages" ,"nr_kernel_stack" , +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +"nr_shadow_call_stack_bytes" , +#endif"nr_bounce" , #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages",
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 3/12 scs: add support for stack usage debugging
In reply to Sami TolvanenImplements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.
--- kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static inline unsigned long scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static DEFINE_SPINLOCK (lock); +static unsigned long highest; +unsigned long used =scs_used (tsk); + +if (used <= highest) +return ; + +spin_lock (&lock); + +if (used > highest) { +pr_info ("%s (%d): highest shadow stack usage: %lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); + highest = used; + } + +spin_unlock (&lock); +} +#else +static inline void scs_check_usage (struct task_struct *tsk) +{ +} +#endif +bool scs_corrupted (struct task_struct *tsk) {unsigned long *magic =scs_magic (__scs_base (tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)return ;WARN_ON (scs_corrupted (tsk)); +scs_check_usage (tsk);scs_account (tsk,-1 );task_set_scs (tsk,NULL );
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 4/12 scs: disable when function graph tracing is enabled
In reply to Sami TolvanenThe graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
@@ -535,6 +535,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 5/12 arm64: reserve x18 from general allocation with SCS
In reply to Sami TolvanenReserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.
--- arch/arm64/Makefile | 4 ++++ 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
@@ -65,6 +65,10 @@ stack_protector_prepare: prepare0 include/generated/asm-offsets.h)) endif +ifeq ($(CONFIG_SHADOW_CALL_STACK), y) +KBUILD_CFLAGS += -ffixed-x18 +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 6/12 arm64: preserve x18 when CPU is suspended
In reply to Sami TolvanenDon't lose the current task's shadow stack when the CPU is suspended.
--- arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/mm/proc.S | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@#ifndef __ASM_SUSPEND_H#define __ASM_SUSPEND_H -#define NR_CTX_REGS 12 +#define NR_CTX_REGS 13#define NR_CALLEE_SAVED_REGS 12/*
arch/arm64/mm/proc.S +14 -0
@@ -56,6 +56,8 @@ * cpu_do_suspend - save CPU registers context * * x0: virtual address of context pointer + * + * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. */ SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0
@@ -80,6 +82,11 @@ alternative_endif stp x8, x9, [x0, #48] stp x10, x11, [x0, #64] stp x12, x13, [x0, #80] + /* + * Save x18 as it may be used as a platform register, e.g. by shadow + * call stack. + */ + str x18, [x0, #96] ret SYM_FUNC_END(cpu_do_suspend)
@@ -96,6 +103,13 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + /* + * Restore x18, as it may be used as a platform register, and clear + * the buffer to minimize the risk of exposure when used for shadow + * call stack. + */ + ldr x18, [x0, #96] + str xzr, [x0, #96] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 7/12 arm64: efi: restore x18 if it was corrupted
In reply to Sami TolvanenIf we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.
--- arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper) ldp x29, x30, [sp], #32 b.ne 0f ret -0: b efi_handle_corrupted_x18 // tail call +0: + /* + * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a + * shadow stack pointer, which we need to restore before returning to + * potentially instrumented code. This is safe because the wrapper is + * called with preemption disabled and a separate shadow stack is used + * for interrupts. + */ + mov x18, x2 + b efi_handle_corrupted_x18 // tail call ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 8/12 arm64: vdso: disable Shadow Call Stack
In reply to Sami TolvanenShadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.
--- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING VDSO_LDFLAGS := -Bsymbolic -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) KBUILD_CFLAGS += $(DISABLE_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 9/12 arm64: disable SCS for hypervisor code
In reply to Sami TolvanenDisable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.
--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 10/12 arm64: implement Shadow Call Stack
In reply to Sami TolvanenThis change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
--- arch/arm64/Kconfig | 5 ++++ arch/arm64/include/asm/scs.h | 37 +++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 3 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 32 ++++++++++++++++++++-- arch/arm64/kernel/head.S | 9 +++++++ arch/arm64/kernel/irq.c | 2 ++ arch/arm64/kernel/process.c | 2 ++ arch/arm64/kernel/scs.c | 40 ++++++++++++++++++++++++++++ arch/arm64/kernel/smp.c | 4 +++ 11 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
@@ -65,6 +65,7 @@ config ARM64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1022,6 +1023,10 @@ config ARCH_HAS_CACHE_LINE_SIZE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 +# Supported by clang >= 7.0 +config CC_HAVE_SHADOW_CALL_STACK + def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help---
arch/arm64/include/asm/scs.h new file +37 -0
@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifndef __ASSEMBLY__ + +#include <linux/scs.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +extern void scs_init_irq (void ); + +static __always_inline void scs_save(struct task_struct *tsk) +{ +void *s; + + asmvolatile ("mov %0, x18" :"=r" (s)); +task_set_scs (tsk, s); +} + +static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" ); +} + +#else /* CONFIG_SHADOW_CALL_STACK */ + +static inline void scs_init_irq (void ) {} +static inline void scs_save (struct task_struct *tsk) {} +static inline void scs_overflow_check (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* __ASSEMBLY __ */ + +#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
@@ -41,6 +41,9 @@ struct thread_info {#endif } preempt; }; +#ifdef CONFIG_SHADOW_CALL_STACK +void *shadow_call_stack; +#endif };#define thread_saved_pc (tsk) \
arch/arm64/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_SSBD) += ssbd.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-y += vdso/ probes/ obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
@@ -33,6 +33,9 @@ int main(void) DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));#ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif +#ifdef CONFIG_SHADOW_CALL_STACK + DEFINE(TSK_TI_SCS, offsetof(struct task_struct, thread_info.shadow_call_stack));#endif DEFINE (TSK_STACK , offsetof(struct task_struct , stack));#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +30 -2
@@ -177,6 +177,10 @@ alternative_cb_end apply_ssbd 1, x22, x23 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [tsk, #TSK_TI_SCS] // Restore shadow call stack + str xzr, [tsk, #TSK_TI_SCS] // Limit visibility of saved SCS +#endif .else add x21, sp, #S_FRAME_SIZE get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif ct_user_enter .endif +#ifdef CONFIG_SHADOW_CALL_STACK + .if \el == 0 + str x18, [tsk, #TSK_TI_SCS] // Save shadow call stack + .endif +#endif + #ifdef CONFIG_ARM64_SW_TTBR0_PAN /* * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .macro irq_stack_entry mov x19, sp // preserve the original sp +#ifdef CONFIG_SHADOW_CALL_STACK + mov x24, x18 // preserve the original shadow stack +#endif /* * Compare sp with the base of the task stack.
@@ -400,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 /* switch to the irq stack */ mov sp, x26 + +#ifdef CONFIG_SHADOW_CALL_STACK + /* also switch to the irq shadow stack */ + ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26 +#endif + 9998: .endm /* - * x19 should be preserved between irq_stack_entry and - * irq_stack_exit. + * The callee-saved regs (x19-x29) should be preserved between + * irq_stack_entry and irq_stack_exit, but note that kernel_entry + * uses x20-x23 to store data for later use. */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_SHADOW_CALL_STACK + mov x18, x24 +#endif .endm /* GPRs used by entry code */
@@ -895,6 +918,11 @@ ENTRY(cpu_switch_to) ldr lr, [x8] mov sp, x9 msr sp_el0, x1 +#ifdef CONFIG_SHADOW_CALL_STACK + str x18, [x0, #TSK_TI_SCS] + ldr x18, [x1, #TSK_TI_SCS] + str xzr, [x1, #TSK_TI_SCS] // limit visibility of saved SCS +#endif ret ENDPROC(cpu_switch_to) NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
@@ -27,6 +27,7 @@ #include <asm/pgtable-hwdef.h> #include <asm/pgtable.h> #include <asm/page.h> +#include <asm/scs.h> #include <asm/smp.h> #include <asm/sysreg.h> #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched: stp xzr, x30, [sp, #-16]! mov x29, sp +#ifdef CONFIG_SHADOW_CALL_STACK + adr_l x18, init_shadow_call_stack // Set shadow call stack +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched: ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow msr sp_el0, x2 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [x2, #TSK_TI_SCS] // set shadow call stack + str xzr, [x2, #TSK_TI_SCS] // limit visibility of saved SCS +#endif mov x29, #0 mov x30, #0 b secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
@@ -21,6 +21,7 @@#include <linux/vmalloc.h> #include <asm/daifflags.h> #include <asm/vmap_stack.h> +#include <asm/scs.h> unsigned long irq_err_count;
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)void __init init_IRQ (void ) {init_irq_stacks (); +scs_init_irq ();irqchip_init ();if (!handle_arch_irq)panic ("No interrupt controller found." );
arch/arm64/kernel/process.c +2 -0
@@ -52,6 +52,7 @@#include <asm/mmu_context.h> #include <asm/processor.h> #include <asm/pointer_auth.h> +#include <asm/scs.h> #include <asm/stacktrace.h> #if defined(CONFIG_STACKPROTECTOR ) && !defined(CONFIG_STACKPROTECTOR_PER_TASK )
@@ -514,6 +515,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,uao_thread_switch (next);ptrauth_thread_switch (next);ssbs_thread_switch (next); +scs_overflow_check (next);/* * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
@@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/percpu.h> +#include <linux/vmalloc.h> +#include <asm/pgtable.h> +#include <asm/scs.h> + +DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); + +#ifndef CONFIG_SHADOW_CALL_STACK_VMAP +DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack) +__aligned(SCS_SIZE) ; +#endif + +void scs_init_irq (void ) +{ +int cpu; + +for_each_possible_cpu (cpu) { +#ifdef CONFIG_SHADOW_CALL_STACK_VMAP +unsigned long *p; + + p =__vmalloc_node_range (PAGE_SIZE ,SCS_SIZE , +VMALLOC_START ,VMALLOC_END , +GFP_SCS ,PAGE_KERNEL , +0 ,cpu_to_node (cpu), +__builtin_return_address (0 )); + +per_cpu (irq_shadow_call_stack_ptr, cpu) = p; +#else +per_cpu (irq_shadow_call_stack_ptr, cpu) = +per_cpu (irq_shadow_call_stack, cpu); +#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */ + } +}
arch/arm64/kernel/smp.c +4 -0
@@ -46,6 +46,7 @@#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/processor.h> +#include <asm/scs.h> #include <asm/smp_plat.h> #include <asm/sections.h> #include <asm/tlbflush.h>
@@ -358,6 +359,9 @@ void cpu_die(void) {unsigned int cpu =smp_processor_id (); +/* Save the shadow stack pointer before exiting the idle task */ +scs_save (current); +idle_task_exit ();local_daif_mask ();
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Re: [PATCH v9 10/12] arm64: implement Shadow Call Stack
Hi Sami,
Quoted 9 lines · Sami Tolvanen
This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
arch/arm64/include/asm/scs.h new file +0
-0
arch/arm64/include/asm/scs.h line 5
Quoted 5 lines
arch/arm64/include/asm/scs.h new file +5
-0@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifndef __ASSEMBLY__
As the whole file is guarded by this, why do you need to include it in assembly files at all?
Quoted 9 lines
arch/arm64/include/asm/scs.h new file +9
-0+ +#include <linux/scs.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +extern void scs_init_irq (void ); + +static __always_inline void scs_save(struct task_struct *tsk) +{
arch/arm64/include/asm/scs.h line 24
Quoted 10 lines
arch/arm64/include/asm/scs.h new file +10
-0+void *s; + + asmvolatile ("mov %0, x18" :"=r" (s)); +task_set_scs (tsk, s); +} + +static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" );
Could this ever catch anything with CONFIG_SHADOW_CALL_STACK_VMAP? Wouldn't we have hit the vmalloc guard page at the point of overflow?
arch/arm64/kernel/entry.S
Quoted 20 lines
arch/arm64/include/asm/scs.h new file +13
-0+} + +#else /* CONFIG_SHADOW_CALL_STACK */ + +static inline void scs_init_irq (void ) {} +static inline void scs_save (struct task_struct *tsk) {} +static inline void scs_overflow_check (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* __ASSEMBLY __ */ + +#endif /* _ASM_SCS_H */ arch/arm64/kernel/entry.S +0
-0
If I corrupt x18 so that we take an exception (mov x18, xzr), we take that exception whenever we run C code. The CPU 'vanishes' and I get a very upset scheduler shortly after.
Stack misalignment has the same problem, but the overflow test (eventually) catches that, then calls panic() using the overflow stack. (See the kernel_ventry macro and __bad_stack in entry.S)
It would be nice to have a per-cpu stack that we switch to when on the overflow stack. (this would catch the scs overflow hitting the guard page too, as we should eat through the regular stack until we overflowed it!)
arch/arm64/kernel/smp.c void cpu_die(void) · line 362
Quoted 9 lines
arch/arm64/kernel/smp.c +1
-0@@ -358,6 +359,9 @@ void cpu_die(void) {unsigned int cpu =smp_processor_id (); +/* Save the shadow stack pointer before exiting the idle task */
I can't work out why this needs to be before before idle_task_exit()...
It needs to run before init_idle(), which calls scs_task_reset(), but all that is on the cpu_up() path. (if it is to pair those up, any reason core code can't do both?)
arch/arm64/kernel/smp.c void cpu_die(void) · line 367
Quoted 6 lines
arch/arm64/kernel/smp.c +2
-0+scs_save (current); +idle_task_exit ();local_daif_mask ();
Thanks!
James
Re: [PATCH v9 10/12] arm64: implement Shadow Call Stack
Quoted 3 lines · James Morse
Quoted 1 line
+#ifndef
__ASSEMBLY__As the whole file is guarded by this, why do you need to include it in assembly files at all?
True, the include in head.S is not needed. I'll remove it in the next version.
Quoted 7 lines
arch/arm64/include/asm/scs.h line 24
Quoted 4 lines
arch/arm64/include/asm/scs.h new file +4
-0+static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" );Could this ever catch anything with
CONFIG_SHADOW_CALL_STACK_VMAP? Wouldn't we have hit the vmalloc guard page at the point of overflow?
With CONFIG_SHADOW_CALL_STACK_VMAP, even though we allocate a full page, SCS_SIZE is still 1k, so we should catch overflows here well before we hit the guard page.
Quoted 1 line
It would be nice to have a per-cpu stack that we switch to when on the overflow stack.
It shouldn't be a problem to add an overflow shadow stack if you think one is needed.
Quoted 3 lines
I can't work out why this needs to be before before
idle_task_exit()...
It needs to run beforeinit_idle(), which callsscs_task_reset(), but all that is on thecpu_up()path. (if it is to pair those up, any reason core code can't do both?)
At this point, the idle task's shadow stack pointer is only stored in x18, so we need to save it again to thread_info before the CPU shuts down, or we'll lose the pointer.
Sami
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH v9 11/12 arm64: scs: add shadow stacks for SDEI
In reply to Sami TolvanenThis change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.
--- arch/arm64/include/asm/scs.h | 2 + arch/arm64/kernel/entry.S | 14 ++++- arch/arm64/kernel/scs.c | 106 +++++++++++++++++++++++++++++------ arch/arm64/kernel/sdei.c | 7 +++ 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
@@ -9,6 +9,7 @@#ifdef CONFIG_SHADOW_CALL_STACK extern void scs_init_irq (void ); +extern int scs_init_sdei (void ); static __always_inline void scs_save(struct task_struct *tsk) {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)#else /* CONFIG_SHADOW_CALL_STACK */ static inline void scs_init_irq (void ) {} +static inline int scs_init_sdei (void ) {return 0 ; }static inline void scs_save (struct task_struct *tsk) {}static inline void scs_overflow_check (struct task_struct *tsk) {}
arch/arm64/kernel/entry.S +13 -1
@@ -1050,13 +1050,16 @@ ENTRY(__sdei_asm_handler) mov x19, x1 +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] +#endif + #ifdef CONFIG_VMAP_STACK /* * entry.S may have been using sp as a scratch register, find whether * this is a normal or critical event and switch to the appropriate * stack for this CPU. */ - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] cbnz w4, 1f ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 b 2f
@@ -1066,6 +1069,15 @@ ENTRY(__sdei_asm_handler) mov sp, x5 #endif +#ifdef CONFIG_SHADOW_CALL_STACK + /* Use a separate shadow call stack for normal and critical events */ + cbnz w4, 3f + ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 + b 4f +3: ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 +4: +#endif + /* * We may have interrupted userspace, or a guest, or exit-from or * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16
121 lines not on this page · raw message
arch/arm64/kernel/sdei.c +7 -0
@@ -13,6 +13,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/ptrace.h> +#include <asm/scs.h> #include <asm/sections.h> #include <asm/stacktrace.h> #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)return 0 ; } +if (scs_init_sdei ()) { +if (IS_ENABLED (CONFIG_VMAP_STACK )) +free_sdei_stacks (); +return 0 ; + } + sdei_exit_mode = (conduit ==SMCCC_CONDUIT_HVC ) ?SDEI_EXIT_HVC :SDEI_EXIT_SMC ;#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
PATCH v9 12/12 efi/libstub: disable SCS
In reply to Sami TolvanenShadow stacks are not available in the EFI stub, filter out SCS flags.
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -30,6 +30,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
v10 cover add support for Clang's Shadow Call Stack
In reply to Sami TolvanenThis patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.
Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.
Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.
Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.
Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler
Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information
Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical
Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit
Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions
Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS
Makefile | 6 + arch/Kconfig | 35 ++++ arch/arm64/Kconfig | 5 + arch/arm64/Makefile | 4 + arch/arm64/include/asm/kvm_hyp.h | 2 +- arch/arm64/include/asm/scs.h | 39 ++++ arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/include/asm/thread_info.h | 3 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 + arch/arm64/kernel/efi-rt-wrapper.S | 11 +- arch/arm64/kernel/entry.S | 47 ++++- arch/arm64/kernel/head.S | 8 + arch/arm64/kernel/irq.c | 2 + arch/arm64/kernel/process.c | 2 + arch/arm64/kernel/scs.c | 114 ++++++++++++ arch/arm64/kernel/sdei.c | 7 + arch/arm64/kernel/smp.c | 4 + arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/mm/proc.S | 14 ++ drivers/base/node.c | 6 + drivers/firmware/efi/libstub/Makefile | 3 + fs/proc/meminfo.c | 4 + include/linux/compiler-clang.h | 6 + include/linux/compiler_types.h | 4 + include/linux/mmzone.h | 3 + include/linux/scs.h | 57 ++++++ init/init_task.c | 8 + kernel/Makefile | 1 + kernel/fork.c | 9 + kernel/sched/core.c | 2 + kernel/scs.c | 246 ++++++++++++++++++++++++++ mm/page_alloc.c | 6 + mm/vmstat.c | 3 + 34 files changed, 662 insertions(+), 7 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
base-commit: a10c9c710f9ecea87b9f4bbb837467893b4bef01
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 1/12 add support for Clang's Shadow Call Stack (SCS)
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
--- Makefile | 6 ++ arch/Kconfig | 34 ++++++ include/linux/compiler-clang.h | 6 ++ include/linux/compiler_types.h | 4 + include/linux/scs.h | 57 ++++++++++ init/init_task.c | 8 ++ kernel/Makefile | 1 + kernel/fork.c | 9 ++ kernel/sched/core.c | 2 + kernel/scs.c | 187 +++++++++++++++++++++++++++++++++ 10 files changed, 314 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
Makefile +6 -0
@@ -851,6 +851,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
arch/Kconfig +34 -0
@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use. + +config SHADOW_CALL_STACK_VMAP + bool "Use virtually mapped shadow call stacks" + depends on SHADOW_CALL_STACK + help + Use virtually mapped shadow call stacks. Selecting this option + provides better stack exhaustion protection, but increases per-thread + memory consumption as a full page is allocated for each shadow stack. + + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help
include/linux/compiler-clang.h +6 -0
@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif
include/linux/compiler_types.h +4 -0
@@ -202,6 +202,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif
include/linux/scs.h new file +57 -0
57 lines not on this page · raw message
init/init_task.c +8 -0
@@ -11,6 +11,7 @@#include <linux/mm.h> #include <linux/audit.h> #include <linux/numa.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task };EXPORT_SYMBOL (init_task); +#ifdef CONFIG_SHADOW_CALL_STACK +unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data +__aligned(SCS_SIZE) = { + [(SCS_SIZE /sizeof (long )) -1 ] =SCS_END_MAGIC +}; +#endif +/* * Initial thread structure. Alignment of this is handled by a special * linker map entry.
kernel/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/fork.c +9 -0
@@ -94,6 +94,7 @@#include <linux/thread_info.h> #include <linux/stackleak.h> #include <linux/kasan.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
@@ -455,6 +456,8 @@ void put_task_stack(struct task_struct *tsk)void free_task (struct task_struct *tsk) { +scs_release (tsk); +#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info,
@@ -838,6 +841,8 @@ void __init fork_init(void) NULL, free_vm_stack_cache);#endif +scs_init (); +lockdep_init_task (&init_task);uprobes_init (); }
@@ -897,6 +902,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)if (err) goto free_stack; + err =scs_prepare (tsk, node); +if (err) + goto free_stack; +#ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h>
@@ -6050,6 +6051,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP
kernel/scs.c new file +187 -0
187 lines not on this page · raw message
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 2/12 scs: add accounting
In reply to Sami TolvanenThis change adds accounting for the memory allocated for shadow stacks.
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 20 ++++++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid,sum_zone_node_page_state (nid,NR_KERNEL_SCS_BYTES ) /1024 , +#endif nid,K (sum_zone_node_page_state (nid,NR_PAGETABLE )), nid,K (node_page_state (pgdat,NR_UNSTABLE_NFS )), nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)show_val_kb (m,"SUnreclaim: " , sunreclaim);seq_printf (m,"KernelStack: %8lu kB\n" ,global_zone_page_state (NR_KERNEL_STACK_KB )); +#ifdef CONFIG_SHADOW_CALL_STACK +seq_printf (m,"ShadowCallStack:%8lu kB\n" , +global_zone_page_state (NR_KERNEL_SCS_BYTES ) /1024 ); +#endif show_val_kb (m,"PageTables: " ,global_zone_page_state (NR_PAGETABLE ));
include/linux/mmzone.h +3 -0
@@ -200,6 +200,9 @@ enum zone_stat_item {NR_MLOCK ,/* mlock()ed pages found and moved off LRU */ NR_PAGETABLE ,/* used for pagetables */ NR_KERNEL_STACK_KB ,/* measured in KiB */ +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +NR_KERNEL_SCS_BYTES ,/* measured in bytes */ +#endif/* Second 128 byte cacheline */ NR_BOUNCE , #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
@@ -12,6 +12,7 @@#include <linux/scs.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/vmstat.h> #include <asm/scs.h> static inline void *__scs_base (struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)vfree_atomic (s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return vmalloc_to_page (__scs_base (tsk)); +} +static int scs_cleanup (unsigned int cpu) {int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)kmem_cache_free (scs_cache, s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return virt_to_page (__scs_base (tsk)); +} +void __init scs_init (void ) { scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)task_set_scs (tsk,__scs_base (tsk)); } +static void scs_account (struct task_struct *tsk,int account) +{ +mod_zone_page_state (page_zone (__scs_page (tsk)),NR_KERNEL_SCS_BYTES , + account *SCS_SIZE ); +} +int scs_prepare (struct task_struct *tsk,int node) {void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)return -ENOMEM ;task_set_scs (tsk, s); +scs_account (tsk,1 ); +return 0 ; }
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)WARN_ON (scs_corrupted (tsk)); +scs_account (tsk,-1 );task_set_scs (tsk,NULL );scs_free (s); }
mm/page_alloc.c +6 -0
@@ -5338,6 +5338,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " managed:%lukB" " mlocked:%lukB" " kernel_stack:%lukB" +#ifdef CONFIG_SHADOW_CALL_STACK + " shadow_call_stack:%lukB" +#endif" pagetables:%lukB" " bounce:%lukB" " free_pcp:%lukB"
@@ -5360,6 +5363,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)K (zone_managed_pages (zone)),K (zone_page_state (zone,NR_MLOCK )),zone_page_state (zone,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK +zone_page_state (zone,NR_KERNEL_SCS_BYTES ) /1024 , +#endifK (zone_page_state (zone,NR_PAGETABLE )),K (zone_page_state (zone,NR_BOUNCE )), K(free_pcp),
mm/vmstat.c +3 -0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {"nr_mlock" ,"nr_page_table_pages" ,"nr_kernel_stack" , +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +"nr_shadow_call_stack_bytes" , +#endif"nr_bounce" , #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages",
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 3/12 scs: add support for stack usage debugging
In reply to Sami TolvanenImplements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.
--- kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static inline unsigned long scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static DEFINE_SPINLOCK (lock); +static unsigned long highest; +unsigned long used =scs_used (tsk); + +if (used <= highest) +return ; + +spin_lock (&lock); + +if (used > highest) { +pr_info ("%s (%d): highest shadow stack usage: %lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); + highest = used; + } + +spin_unlock (&lock); +} +#else +static inline void scs_check_usage (struct task_struct *tsk) +{ +} +#endif +bool scs_corrupted (struct task_struct *tsk) {unsigned long *magic =scs_magic (__scs_base (tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)return ;WARN_ON (scs_corrupted (tsk)); +scs_check_usage (tsk);scs_account (tsk,-1 );task_set_scs (tsk,NULL );
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 4/12 scs: disable when function graph tracing is enabled
In reply to Sami TolvanenThe graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 5/12 arm64: reserve x18 from general allocation with SCS
In reply to Sami TolvanenReserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.
--- arch/arm64/Makefile | 4 ++++ 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
@@ -76,6 +76,10 @@ branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a KBUILD_CFLAGS += $(branch-prot-flags-y) endif +ifeq ($(CONFIG_SHADOW_CALL_STACK), y) +KBUILD_CFLAGS += -ffixed-x18 +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 6/12 arm64: preserve x18 when CPU is suspended
In reply to Sami TolvanenDon't lose the current task's shadow stack when the CPU is suspended.
--- arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/mm/proc.S | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@#ifndef __ASM_SUSPEND_H#define __ASM_SUSPEND_H -#define NR_CTX_REGS 12 +#define NR_CTX_REGS 13#define NR_CALLEE_SAVED_REGS 12/*
arch/arm64/mm/proc.S +14 -0
@@ -58,6 +58,8 @@ * cpu_do_suspend - save CPU registers context * * x0: virtual address of context pointer + * + * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. */ SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif stp x8, x9, [x0, #48] stp x10, x11, [x0, #64] stp x12, x13, [x0, #80] + /* + * Save x18 as it may be used as a platform register, e.g. by shadow + * call stack. + */ + str x18, [x0, #96] ret SYM_FUNC_END(cpu_do_suspend)
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + /* + * Restore x18, as it may be used as a platform register, and clear + * the buffer to minimize the risk of exposure when used for shadow + * call stack. + */ + ldr x18, [x0, #96] + str xzr, [x0, #96] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 7/12 arm64: efi: restore x18 if it was corrupted
In reply to Sami TolvanenIf we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.
--- arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper) ldp x29, x30, [sp], #32 b.ne 0f ret -0: b efi_handle_corrupted_x18 // tail call +0: + /* + * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a + * shadow stack pointer, which we need to restore before returning to + * potentially instrumented code. This is safe because the wrapper is + * called with preemption disabled and a separate shadow stack is used + * for interrupts. + */ + mov x18, x2 + b efi_handle_corrupted_x18 // tail call ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 8/12 arm64: vdso: disable Shadow Call Stack
In reply to Sami TolvanenShadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.
--- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING VDSO_LDFLAGS := -Bsymbolic -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) KBUILD_CFLAGS += $(DISABLE_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 9/12 arm64: disable SCS for hypervisor code
In reply to Sami TolvanenDisable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.
--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Re: [PATCH v10 09/12] arm64: disable SCS for hypervisor code
Quoted 5 lines · Sami Tolvanen
Disable SCS for code that runs at a different exception level by adding
__noscsto__hyp_text.Suggested-by James Morse <[email protected]>Signed-off-by Sami Tolvanen <[email protected]>
-Kees
Quoted 21 lines
Acked-by Marc Zyngier <[email protected]>--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)arch/arm64/include/asm/kvm_hyp.h +1
-1@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Signature · 1 line
Kees Cook
PATCH v10 10/12 arm64: implement Shadow Call Stack
In reply to Sami TolvanenThis change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
--- arch/arm64/Kconfig | 5 ++++ arch/arm64/include/asm/scs.h | 37 +++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 3 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 33 +++++++++++++++++++++-- arch/arm64/kernel/head.S | 8 ++++++ arch/arm64/kernel/irq.c | 2 ++ arch/arm64/kernel/process.c | 2 ++ arch/arm64/kernel/scs.c | 40 ++++++++++++++++++++++++++++ arch/arm64/kernel/smp.c | 4 +++ 11 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
@@ -64,6 +64,7 @@ config ARM64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 +# Supported by clang >= 7.0 +config CC_HAVE_SHADOW_CALL_STACK + def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help---
arch/arm64/include/asm/scs.h new file +37 -0
@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifndef __ASSEMBLY__ + +#include <linux/scs.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +extern void scs_init_irq (void ); + +static __always_inline void scs_save(struct task_struct *tsk) +{ +void *s; + + asmvolatile ("mov %0, x18" :"=r" (s)); +task_set_scs (tsk, s); +} + +static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" ); +} + +#else /* CONFIG_SHADOW_CALL_STACK */ + +static inline void scs_init_irq (void ) {} +static inline void scs_save (struct task_struct *tsk) {} +static inline void scs_overflow_check (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* __ASSEMBLY __ */ + +#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
@@ -41,6 +41,9 @@ struct thread_info {#endif } preempt; }; +#ifdef CONFIG_SHADOW_CALL_STACK +void *shadow_call_stack; +#endif };#define thread_saved_pc (tsk) \
arch/arm64/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_SSBD) += ssbd.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-y += vdso/ probes/ obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
@@ -33,6 +33,9 @@ int main(void) DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));#ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif +#ifdef CONFIG_SHADOW_CALL_STACK + DEFINE(TSK_TI_SCS, offsetof(struct task_struct, thread_info.shadow_call_stack));#endif DEFINE (TSK_STACK , offsetof(struct task_struct , stack));#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
@@ -179,6 +179,11 @@ alternative_cb_end apply_ssbd 1, x22, x23 ptrauth_keys_install_kernel tsk, 1, x20, x22, x23 + +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [tsk, #TSK_TI_SCS] // Restore shadow call stack + str xzr, [tsk, #TSK_TI_SCS] // Limit visibility of saved SCS +#endif .else add x21, sp, #S_FRAME_SIZE get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif ct_user_enter .endif +#ifdef CONFIG_SHADOW_CALL_STACK + .if \el == 0 + str x18, [tsk, #TSK_TI_SCS] // Save shadow call stack + .endif +#endif + #ifdef CONFIG_ARM64_SW_TTBR0_PAN /* * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .macro irq_stack_entry mov x19, sp // preserve the original sp +#ifdef CONFIG_SHADOW_CALL_STACK + mov x24, x18 // preserve the original shadow stack +#endif /* * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 /* switch to the irq stack */ mov sp, x26 + +#ifdef CONFIG_SHADOW_CALL_STACK + /* also switch to the irq shadow stack */ + ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26 +#endif + 9998: .endm /* - * x19 should be preserved between irq_stack_entry and - * irq_stack_exit. + * The callee-saved regs (x19-x29) should be preserved between + * irq_stack_entry and irq_stack_exit, but note that kernel_entry + * uses x20-x23 to store data for later use. */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_SHADOW_CALL_STACK + mov x18, x24 +#endif .endm /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to) mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, 1, x8, x9, x10 +#ifdef CONFIG_SHADOW_CALL_STACK + str x18, [x0, #TSK_TI_SCS] + ldr x18, [x1, #TSK_TI_SCS] + str xzr, [x1, #TSK_TI_SCS] // limit visibility of saved SCS +#endif ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) stp xzr, x30, [sp, #-16]! mov x29, sp +#ifdef CONFIG_SHADOW_CALL_STACK + adr_l x18, init_shadow_call_stack // Set shadow call stack +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched) ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow msr sp_el0, x2 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [x2, #TSK_TI_SCS] // set shadow call stack + str xzr, [x2, #TSK_TI_SCS] // limit visibility of saved SCS +#endif mov x29, #0 mov x30, #0 b secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
@@ -21,6 +21,7 @@#include <linux/vmalloc.h> #include <asm/daifflags.h> #include <asm/vmap_stack.h> +#include <asm/scs.h> unsigned long irq_err_count;
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)void __init init_IRQ (void ) {init_irq_stacks (); +scs_init_irq ();irqchip_init ();if (!handle_arch_irq)panic ("No interrupt controller found." );
arch/arm64/kernel/process.c +2 -0
@@ -52,6 +52,7 @@#include <asm/mmu_context.h> #include <asm/processor.h> #include <asm/pointer_auth.h> +#include <asm/scs.h> #include <asm/stacktrace.h> #if defined(CONFIG_STACKPROTECTOR ) && !defined(CONFIG_STACKPROTECTOR_PER_TASK )
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,entry_task_switch (next);uao_thread_switch (next);ssbs_thread_switch (next); +scs_overflow_check (next);/* * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
@@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/percpu.h> +#include <linux/vmalloc.h> +#include <asm/pgtable.h> +#include <asm/scs.h> + +DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); + +#ifndef CONFIG_SHADOW_CALL_STACK_VMAP +DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack) +__aligned(SCS_SIZE) ; +#endif + +void scs_init_irq (void ) +{ +int cpu; + +for_each_possible_cpu (cpu) { +#ifdef CONFIG_SHADOW_CALL_STACK_VMAP +unsigned long *p; + + p =__vmalloc_node_range (PAGE_SIZE ,SCS_SIZE , +VMALLOC_START ,VMALLOC_END , +GFP_SCS ,PAGE_KERNEL , +0 ,cpu_to_node (cpu), +__builtin_return_address (0 )); + +per_cpu (irq_shadow_call_stack_ptr, cpu) = p; +#else +per_cpu (irq_shadow_call_stack_ptr, cpu) = +per_cpu (irq_shadow_call_stack, cpu); +#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */ + } +}
arch/arm64/kernel/smp.c +4 -0
@@ -46,6 +46,7 @@#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/processor.h> +#include <asm/scs.h> #include <asm/smp_plat.h> #include <asm/sections.h> #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)unsigned int cpu =smp_processor_id ();const struct cpu_operations *ops =get_cpu_ops (cpu); +/* Save the shadow stack pointer before exiting the idle task */ +scs_save (current); +idle_task_exit ();local_daif_mask ();
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 11/12 arm64: scs: add shadow stacks for SDEI
In reply to Sami TolvanenThis change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.
--- arch/arm64/include/asm/scs.h | 2 + arch/arm64/kernel/entry.S | 14 ++++- arch/arm64/kernel/scs.c | 106 +++++++++++++++++++++++++++++------ arch/arm64/kernel/sdei.c | 7 +++ 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
@@ -9,6 +9,7 @@#ifdef CONFIG_SHADOW_CALL_STACK extern void scs_init_irq (void ); +extern int scs_init_sdei (void ); static __always_inline void scs_save(struct task_struct *tsk) {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)#else /* CONFIG_SHADOW_CALL_STACK */ static inline void scs_init_irq (void ) {} +static inline int scs_init_sdei (void ) {return 0 ; }static inline void scs_save (struct task_struct *tsk) {}static inline void scs_overflow_check (struct task_struct *tsk) {}
arch/arm64/kernel/entry.S +13 -1
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler) mov x19, x1 +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] +#endif + #ifdef CONFIG_VMAP_STACK /* * entry.S may have been using sp as a scratch register, find whether * this is a normal or critical event and switch to the appropriate * stack for this CPU. */ - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] cbnz w4, 1f ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 b 2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler) mov sp, x5 #endif +#ifdef CONFIG_SHADOW_CALL_STACK + /* Use a separate shadow call stack for normal and critical events */ + cbnz w4, 3f + ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 + b 4f +3: ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 +4: +#endif + /* * We may have interrupted userspace, or a guest, or exit-from or * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16
121 lines not on this page · raw message
arch/arm64/kernel/sdei.c +7 -0
@@ -13,6 +13,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/ptrace.h> +#include <asm/scs.h> #include <asm/sections.h> #include <asm/stacktrace.h> #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)return 0 ; } +if (scs_init_sdei ()) { +if (IS_ENABLED (CONFIG_VMAP_STACK )) +free_sdei_stacks (); +return 0 ; + } + sdei_exit_mode = (conduit ==SMCCC_CONDUIT_HVC ) ?SDEI_EXIT_HVC :SDEI_EXIT_SMC ;#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
PATCH v10 12/12 efi/libstub: disable SCS
In reply to Sami TolvanenShadow stacks are not available in the EFI stub, filter out SCS flags.
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Re: [PATCH v10 12/12] efi/libstub: disable SCS
Quoted 4 lines · Sami Tolvanen
Shadow stacks are not available in the EFI stub, filter out SCS flags.
Suggested-by James Morse <[email protected]>Signed-off-by Sami Tolvanen <[email protected]>
drivers/firmware/efi/libstub/Makefile KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ · line 35
Quoted 13 lines
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)drivers/firmware/efi/libstub/Makefile +1
-0@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory
nit: double space
-Kees
Quoted 8 lines
drivers/firmware/efi/libstub/Makefile +2
-0+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := nSignature · 1 line
2.26.0.292.g33ef6b2f38-goog
Signature · 6 lines
Kees Cook _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Re: [PATCH v10 12/12] efi/libstub: disable SCS
In reply to Sami TolvanenQuoted 5 lines · Sami Tolvanen
Shadow stacks are not available in the EFI stub, filter out SCS flags.
Suggested-by James Morse <[email protected]>Signed-off-by Sami Tolvanen <[email protected]>
Quoted 21 lines
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)drivers/firmware/efi/libstub/Makefile +3
-1@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n --2.26.0.292.g33ef6b2f38-goog
v11 cover add support for Clang's Shadow Call Stack
In reply to Sami TolvanenThis patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Changes in v11:
- Rebased, added maintainers for kernel/ changes.
Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.
Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.
Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.
Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.
Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler
Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information
Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical
Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit
Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions
Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS
Makefile | 6 + arch/Kconfig | 35 ++++ arch/arm64/Kconfig | 5 + arch/arm64/Makefile | 4 + arch/arm64/include/asm/kvm_hyp.h | 2 +- arch/arm64/include/asm/scs.h | 39 ++++ arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/include/asm/thread_info.h | 3 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 + arch/arm64/kernel/efi-rt-wrapper.S | 11 +- arch/arm64/kernel/entry.S | 47 ++++- arch/arm64/kernel/head.S | 8 + arch/arm64/kernel/irq.c | 2 + arch/arm64/kernel/process.c | 2 + arch/arm64/kernel/scs.c | 114 ++++++++++++ arch/arm64/kernel/sdei.c | 7 + arch/arm64/kernel/smp.c | 4 + arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/mm/proc.S | 14 ++ drivers/base/node.c | 6 + drivers/firmware/efi/libstub/Makefile | 3 + fs/proc/meminfo.c | 4 + include/linux/compiler-clang.h | 6 + include/linux/compiler_types.h | 4 + include/linux/mmzone.h | 3 + include/linux/scs.h | 57 ++++++ init/init_task.c | 8 + kernel/Makefile | 1 + kernel/fork.c | 9 + kernel/sched/core.c | 2 + kernel/scs.c | 246 ++++++++++++++++++++++++++ mm/page_alloc.c | 6 + mm/vmstat.c | 3 + 34 files changed, 662 insertions(+), 7 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
base-commit: 00086336a8d96a04aa960f912287692a258f6cf5
Signature · 7 lines
2.26.1.301.g55bc3eb7cb9-goog _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH v11 1/12 add support for Clang's Shadow Call Stack (SCS)
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
--- Makefile | 6 ++ arch/Kconfig | 34 ++++++ include/linux/compiler-clang.h | 6 ++ include/linux/compiler_types.h | 4 + include/linux/scs.h | 57 ++++++++++ init/init_task.c | 8 ++ kernel/Makefile | 1 + kernel/fork.c | 9 ++ kernel/sched/core.c | 2 + kernel/scs.c | 187 +++++++++++++++++++++++++++++++++ 10 files changed, 314 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
Makefile +6 -0
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
arch/Kconfig +34 -0
@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use. + +config SHADOW_CALL_STACK_VMAP + bool "Use virtually mapped shadow call stacks" + depends on SHADOW_CALL_STACK + help + Use virtually mapped shadow call stacks. Selecting this option + provides better stack exhaustion protection, but increases per-thread + memory consumption as a full page is allocated for each shadow stack. + + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help
include/linux/compiler-clang.h +6 -0
@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif
include/linux/compiler_types.h +4 -0
@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif
include/linux/scs.h new file +57 -0
57 lines not on this page · raw message
init/init_task.c +8 -0
@@ -11,6 +11,7 @@#include <linux/mm.h> #include <linux/audit.h> #include <linux/numa.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task };EXPORT_SYMBOL (init_task); +#ifdef CONFIG_SHADOW_CALL_STACK +unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data +__aligned(SCS_SIZE) = { + [(SCS_SIZE /sizeof (long )) -1 ] =SCS_END_MAGIC +}; +#endif +/* * Initial thread structure. Alignment of this is handled by a special * linker map entry.
kernel/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/fork.c +9 -0
@@ -94,6 +94,7 @@#include <linux/thread_info.h> #include <linux/stackleak.h> #include <linux/kasan.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)void free_task (struct task_struct *tsk) { +scs_release (tsk); +#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void) NULL, free_vm_stack_cache);#endif +scs_init (); +lockdep_init_task (&init_task);uprobes_init (); }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)if (err) goto free_stack; + err =scs_prepare (tsk, node); +if (err) + goto free_stack; +#ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP
kernel/scs.c new file +187 -0
187 lines not on this page · raw message
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Will Deacon
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Hi Sami,
Comments inline.
Quoted 36 lines · Sami Tolvanen
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>Reviewed-by Miguel Ojeda <[email protected]>--- Makefile | 6 ++ arch/Kconfig | 34 ++++++ include/linux/compiler-clang.h | 6 ++ include/linux/compiler_types.h | 4 + include/linux/scs.h | 57 ++++++++++ init/init_task.c | 8 ++ kernel/Makefile | 1 + kernel/fork.c | 9 ++ kernel/sched/core.c | 2 + kernel/scs.c | 187 +++++++++++++++++++++++++++++++++ 10 files changed, 314 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.cMakefile +0
-0
Makefile ifdef CONFIG_LIVEPATCH · line 873
Quoted 8 lines
Makefile +5
-0@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif
CFLAGS_SCS would seem more natural to me, although I see ftrace does it this way.
Quoted 25 lines
Makefile +0
-0# arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)arch/Kconfig +14
-0@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in
arch/Kconfig config STACKPROTECTOR_STRONG · line 559
Quoted 10 lines
arch/Kconfig +10
-0+ Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use.
Shouldn't some of this depend on CC_IS_CLANG?
arch/Kconfig config STACKPROTECTOR_STRONG · line 567
Quoted 7 lines
arch/Kconfig +7
-0+config SHADOW_CALL_STACK_VMAP + bool "Use virtually mapped shadow call stacks" + depends on SHADOW_CALL_STACK + help + Use virtually mapped shadow call stacks. Selecting this option + provides better stack exhaustion protection, but increases per-thread + memory consumption as a full page is allocated for each shadow stack.
Given that this feature applies only to arm64 kernels built with clang, it feels weird to further segment that userbase with another config option. Does Android enable SHADOW_CALL_STACK_VMAP? If not, maybe we should ditch it for now and add it when we have a user.
Quoted 40 lines
include/linux/compiler_types.h +4
-0@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif include/linux/scs.h new file +19
-0@@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#ifndef _LINUX_SCS_H +#define _LINUX_SCS_H + +#include <linux/gfp.h> +#include <linux/sched.h> +#include <asm/page.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +/* +* In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit +* architecture) provided ~40% safety margin on stack usage while keeping
include/linux/scs.h line 29
Quoted 10 lines
include/linux/scs.h new file +10
-0+* memory allocation overhead reasonable. +*/ +#define SCS_SIZE 1024UL +#define GFP_SCS (GFP_KERNEL | __GFP_ZERO) + +/* +* A random number outside the kernel's virtual address space to mark the +* end of the shadow stack. +*/ +#define SCS_END_MAGIC 0xaf0194819b1635f6UL
This seems like it might be arm64-specific. Why not choose something based off CONFIG_ILLEGAL_POINTER_VALUE (see linux/poison.h)?
Quoted 33 lines
kernel/sched/core.c +2
-0@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h> @@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP kernel/scs.c new file +7
-0@@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ +
kernel/scs.c line 17
Quoted 10 lines
kernel/scs.c new file +10
-0+#include <linux/cpuhotplug.h> +#include <linux/kasan.h> +#include <linux/mm.h> +#include <linux/mmzone.h> +#include <linux/scs.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <asm/scs.h> + +static inline void *__scs_base (struct task_struct *tsk)
Please avoid using 'inline' in C files unless there's a good reason not to let the compiler figure it out.
kernel/scs.c line 20
Quoted 3 lines
kernel/scs.c new file +3
-0+{ +/* +* To minimize risk the of exposure, architectures may clear a
Should be "the risk of exposure".
kernel/scs.c line 30
Quoted 10 lines
kernel/scs.c new file +10
-0+* task's thread_info::shadow_call_stack while that task is +* running, and only save/restore the active shadow call stack +* pointer when the usual register may be clobbered (e.g. across +* context switches). +* +* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));
Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
kernel/scs.c line 40
Quoted 10 lines
kernel/scs.c new file +10
-0+} + +static inline unsigned long *scs_magic (void *s) +{ +return (unsigned long *)(s +SCS_SIZE ) -1 ; +} + +static inline void scs_set_magic (void *s) +{ + *scs_magic (s) =SCS_END_MAGIC ;
You added task_set_scs() for this sort of thing, so I'm not convinced you need this extra helper.
kernel/scs.c line 46
Quoted 6 lines
kernel/scs.c new file +6
-0+} + +#ifdef CONFIG_SHADOW_CALL_STACK_VMAP + +/* Matches NR_CACHED_STACKS for VMAP_STACK */ +#define NR_CACHED_SCS 2
Then they should probably both be derived from the same thing?
kernel/scs.c line 55
Quoted 9 lines
kernel/scs.c new file +9
-0+static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]); + +static void *scs_alloc (int node) +{ +int i; +void *s; + +for (i =0 ; i <NR_CACHED_SCS ; i++) { + s =this_cpu_xchg (scs_cache[i],NULL );
Might be worth a comment about the re-entrancy here.
Quoted 11 lines
kernel/scs.c new file +11
-0+if (s) { +memset (s,0 ,SCS_SIZE ); + goto out; + } + } + +/* +* We allocate a full page for the shadow stack, which should be +* more than we need. Check the assumption nevertheless. +*/ +BUILD_BUG_ON (SCS_SIZE >PAGE_SIZE );
kernel/scs.c line 76
Quoted 10 lines
kernel/scs.c new file +10
-0+ + s =__vmalloc_node_range (PAGE_SIZE ,SCS_SIZE , +VMALLOC_START ,VMALLOC_END , +GFP_SCS ,PAGE_KERNEL ,0 , + node,__builtin_return_address (0 )); + +out: +if (s) +scs_set_magic (s); +/* TODO: poison for KASAN, unpoison in scs_free */
We don't usually commit these. What's missing?
Quoted 14 lines
kernel/scs.c new file +14
-0+ +return s; +} + +static void scs_free (void *s) +{ +int i; + +for (i =0 ; i <NR_CACHED_SCS ; i++) +if (this_cpu_cmpxchg (scs_cache[i],0 , s) ==NULL ) +return ; + +vfree_atomic (s); +}
kernel/scs.c line 100
Quoted 10 lines
kernel/scs.c new file +10
-0+ +static int scs_cleanup (unsigned int cpu) +{ +int i; +void **cache =per_cpu_ptr (scs_cache, cpu); + +for (i =0 ; i <NR_CACHED_SCS ; i++) { +vfree (cache[i]); + cache[i] =NULL ; + }
Hmm, can this run concurrently with another CPU doing a stack allocation with this_cpu_cmpxchg()? It probably works out on arm64 thanks to the use of atomics, but we shouldn't be relying on that in core code.
Quoted 32 lines
kernel/scs.c new file +32
-0+ +return 0 ; +} + +void __init scs_init (void ) +{ +WARN_ON (cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"scs:scs_cache" ,NULL , + scs_cleanup) <0 ); +} + +#else /* !CONFIG_SHADOW_CALL_STACK_VMAP */ + +static struct kmem_cache *scs_cache; + +static inline void *scs_alloc (int node) +{ +void *s; + + s =kmem_cache_alloc_node (scs_cache,GFP_SCS , node); +if (s) { +scs_set_magic (s); +/* +* Poison the allocation to catch unintentional accesses to +* the shadow stack when KASAN is enabled. +*/ +kasan_poison_object_data (scs_cache, s); + } + +return s; +} + +static inline void scs_free (void *s)
kernel/scs.c line 142
Quoted 10 lines
kernel/scs.c new file +10
-0+{ +kasan_unpoison_object_data (scs_cache, s); +kmem_cache_free (scs_cache, s); +} + +void __init scs_init (void ) +{ + scs_cache =kmem_cache_create ("scs_cache" ,SCS_SIZE ,SCS_SIZE , +0 ,NULL ); +WARN_ON (!scs_cache);
Memory allocation failure should be noisy enough without this.
kernel/scs.c line 154
Quoted 12 lines
kernel/scs.c new file +12
-0+} + +#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */ + +void scs_task_reset (struct task_struct *tsk) +{ +/* +* Reset the shadow stack to the base address in case the task +* is reused. +*/ +task_set_scs (tsk,__scs_base (tsk)); +}
Why isn't this in the header?
Quoted 9 lines
kernel/scs.c new file +9
-0+ +int scs_prepare (struct task_struct *tsk,int node) +{ +void *s; + + s =scs_alloc (node); +if (!s) +return -ENOMEM ; +
kernel/scs.c line 173
Quoted 10 lines
kernel/scs.c new file +10
-0+task_set_scs (tsk, s); +return 0 ; +} + +bool scs_corrupted (struct task_struct *tsk) +{ +unsigned long *magic =scs_magic (__scs_base (tsk)); + +return READ_ONCE_NOCHECK (*magic) !=SCS_END_MAGIC ; +}
Same here.
kernel/scs.c line 185
Quoted 12 lines
kernel/scs.c new file +12
-0+ +void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s) +return ; + +WARN_ON (scs_corrupted (tsk)); + +task_set_scs (tsk,NULL );
Aren't we about to free the task here? What does clearing the scs pointer achieve?
Will
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 8 lines · Will Deacon
Makefile ifdef CONFIG_LIVEPATCH · line 873
Quoted 5 lines
Makefile +5
-0+ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif
CFLAGS_SCSwould seem more natural to me, although I see ftrace does it this way.
Right, I followed ftrace's example here.
Quoted 11 lines
Quoted 11 lines
arch/Kconfig +11
-0+config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html +
Quoted 8 lines
arch/Kconfig config STACKPROTECTOR_STRONG · line 559
Quoted 6 lines
arch/Kconfig +6
-0+ Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use.Shouldn't some of this depend on
CC_IS_CLANG?
Sure, I'll add CC_IS_CLANG here in the next version. Note that we do check for compiler support before selecting ARCH_SUPPORTS_SHADOW_CALL_STACK. The flags are architecture-specific, so the check is done in the arch Kconfig.
Quoted 4 lines
Quoted 4 lines
arch/Kconfig +4
-0+config SHADOW_CALL_STACK_VMAP + bool "Use virtually mapped shadow call stacks" + depends on SHADOW_CALL_STACK + help
Quoted 8 lines
arch/Kconfig config STACKPROTECTOR_STRONG · line 567
Quoted 3 lines
arch/Kconfig +3
-0+ Use virtually mapped shadow call stacks. Selecting this option + provides better stack exhaustion protection, but increases per-thread + memory consumption as a full page is allocated for each shadow stack.Given that this feature applies only to arm64 kernels built with clang, it feels weird to further segment that userbase with another config option. Does Android enable
SHADOW_CALL_STACK_VMAP? If not, maybe we should ditch it for now and add it when we have a user.
Android doesn't enable the VMAP option right now due to increased memory overhead. I'll drop it from v12.
Quoted 8 lines
include/linux/scs.h line 29
Quoted 5 lines
include/linux/scs.h new file +5
-0+/* +* A random number outside the kernel's virtual address space to mark the +* end of the shadow stack. +*/ +#define SCS_END_MAGIC 0xaf0194819b1635f6ULThis seems like it might be arm64-specific. Why not choose something based off
CONFIG_ILLEGAL_POINTER_VALUE(seelinux/poison.h)?
Sure, I'll use POISON_POINTER_DELTA here.
Quoted 4 lines
Quoted 1 line
+static inline void *__scs_base(struct
task_struct*tsk)Please avoid using 'inline' in C files unless there's a good reason not to let the compiler figure it out.
Ack.
Quoted 5 lines
kernel/scs.c line 20
Quoted 3 lines
kernel/scs.c new file +3
-0+{ +/* +* To minimize risk the of exposure, architectures may clear a Should be "the risk of exposure".
Thanks.
Quoted 9 lines
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
Quoted 6 lines
kernel/scs.c line 40
Quoted 3 lines
kernel/scs.c new file +3
-0+static inline void scs_set_magic (void *s) +{ + *scs_magic (s) =SCS_END_MAGIC ;You added
task_set_scs()for this sort of thing, so I'm not convinced you need this extra helper.
Agreed, I'll drop this.
Quoted 5 lines
kernel/scs.c line 76
Quoted 3 lines
kernel/scs.c new file +3
-0+if (s) +scs_set_magic (s); +/* TODO: poison for KASAN, unpoison in scs_free */ We don't usually commit these. What's missing?
At the time, KASAN didn't support poisoning vmalloc'ed memory, but looks like that was fixed a while back.
Quoted 5 lines
Quoted 5 lines
kernel/scs.c new file +5
-0+static int scs_cleanup (unsigned int cpu) +{ +int i; +void **cache =per_cpu_ptr (scs_cache, cpu); +
Quoted 8 lines
kernel/scs.c line 100
Quoted 4 lines
kernel/scs.c new file +4
-0+for (i =0 ; i <NR_CACHED_SCS ; i++) { +vfree (cache[i]); + cache[i] =NULL ; + }Hmm, can this run concurrently with another CPU doing a stack allocation with
this_cpu_cmpxchg()? It probably works out on arm64 thanks to the use of atomics, but we shouldn't be relying on that in core code.
This is essentially identical to the code in kernel/fork.c. Anyway, all of this code goes away with the VMAP option.
Quoted 7 lines
kernel/scs.c line 142
Quoted 5 lines
kernel/scs.c new file +5
-0+void __init scs_init (void ) +{ + scs_cache =kmem_cache_create ("scs_cache" ,SCS_SIZE ,SCS_SIZE , +0 ,NULL ); +WARN_ON (!scs_cache);Memory allocation failure should be noisy enough without this.
Sure, I'll remove the warning.
Quoted 11 lines
kernel/scs.c line 154
Quoted 8 lines
kernel/scs.c new file +8
-0+void scs_task_reset (struct task_struct *tsk) +{ +/* +* Reset the shadow stack to the base address in case the task +* is reused. +*/ +task_set_scs (tsk,__scs_base (tsk)); +}Why isn't this in the header?
Quoted 8 lines
kernel/scs.c line 173
Quoted 6 lines
kernel/scs.c new file +6
-0+bool scs_corrupted (struct task_struct *tsk) +{ +unsigned long *magic =scs_magic (__scs_base (tsk)); + +return READ_ONCE_NOCHECK (*magic) !=SCS_END_MAGIC ; +}Same here.
I'll move both to the header file.
Quoted 6 lines
Quoted 6 lines
kernel/scs.c new file +6
-0+void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s)
Quoted 8 lines
kernel/scs.c line 185
Quoted 5 lines
kernel/scs.c new file +5
-0+return ; + +WARN_ON (scs_corrupted (tsk)); + +task_set_scs (tsk,NULL );Aren't we about to free the task here? What does clearing the scs pointer achieve?
True, it doesn't achieve much, only leaves one fewer shadow stack pointer in memory. I'll drop this from the next version.
Sami
Will Deacon
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.
Thanks,
Will
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 20 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
Quoted 4 lines
But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.
It's a hardening step to just reduce the lifetime of a valid address exposed in memory. In fact, since there is a cache, I think it should be wiped even in scs_release().
Signature · 1 line
Kees Cook
Will Deacon
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 21 lines · Kees Cook
Quoted 20 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
Quoted 8 lines
Quoted 4 lines
But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.
It's a hardening step to just reduce the lifetime of a valid address exposed in memory. In fact, since there is a cache, I think it should be wiped even in
scs_release().
But we don't do this for /anything/ else and it forces alignment restrictions on the SCS allocation. Please either do it consistently, or not at all.
Will
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
In reply to Will DeaconQuoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 7 lines
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move __scs_base() logic to cpu_switch_to() / scs_save().
Quoted 4 lines
But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.
Like Kees pointed out, this makes it slightly harder to locate the current task's shadow stack pointer. I realize there are other useful targets in thread_info, but we would rather not make this any easier than necessary. Is your primary concern here the cost of doing this, or just that it doesn't sufficiently improve security?
Sami
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 21 lines · Sami Tolvanen
Quoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 7 lines
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
Quoted 4 lines
I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move
__scs_base()logic tocpu_switch_to()/scs_save().
Okay, so, I feel like this has gotten off into the weeds, or I'm really dense (or both). :) Going back to the original comment:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
I think there was some confusion about mixing the "we want to be able to wipe the value" combined with the masking in __scs_base(). These are unrelated, as was correctly observed with "We can still clear the base".
What I don't understand here is the suggestion to store two values:
Why is two better than storing one? With one, we only need a single access.
Why would storing the base be "friendlier on the allocations later on"? This is coming out of a single kmem cache, in 1K chunks. They will be naturally aligned to 1K (unless redzoing has been turned on for some slab debugging reason). The base masking is a way to avoid needing to store two values, and only happens at task death.
Storing two values eats memory for all tasks for seemingly no meaningful common benefit. What am I missing here?
Signature · 1 line
Kees Cook
Will Deacon
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 30 lines · Kees Cook
Quoted 21 lines · Sami Tolvanen
Quoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon
kernel/scs.c line 30
Quoted 5 lines
kernel/scs.c new file +5
-0+* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 ));Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
Quoted 7 lines
Quoted 2 lines
The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.
What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.
Quoted 4 lines
I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move
__scs_base()logic tocpu_switch_to()/scs_save().Okay, so, I feel like this has gotten off into the weeds, or I'm really dense (or both). :) Going back to the original comment:
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.
I think there was some confusion about mixing the "we want to be able to wipe the value" combined with the masking in
__scs_base(). These are unrelated, as was correctly observed with "We can still clear the base".
Having just tried to implement this, it turns out they *are* related and we can't still clear the base, I was wrong about that :( See below.
Quoted 9 lines
What I don't understand here is the suggestion to store two values:
Why is two better than storing one? With one, we only need a single access.
Why would storing the base be "friendlier on the allocations later on"? This is coming out of a single kmem cache, in 1K chunks. They will be naturally aligned to 1K (unless redzoing has been turned on for some slab debugging reason). The base masking is a way to avoid needing to store two values, and only happens at task death.
Fair enough about the kmem_cache, although I'm still worried about these things getting bigger in future and the alignment having to increase at the same time. We also have a bunch of static/percpu allocations that don't use this cache.
Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS? The vmap version that I asked Sami to drop is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't pass VM_NO_GUARD. Looks like the same story for vmap stack :/
Quoted 2 lines
Storing two values eats memory for all tasks for seemingly no meaningful common benefit. What am I missing here?
I would like to remove the alignment requirements for the static and percpu allocations. AFAICT, the only reason the alignment is needed is because you want to convert an SCS pointer into the base pointer. The only reason *that* is needed is because of the questionable wiping of the pointer in the thread_info, but I really don't see the benefit of this. Unlike a crypto secret (which was your analogy), the SCS pointer is stored in memory in at least the following situations:
* The task isn't running
* The task is running in userspace
* The task is running a vCPU in KVM
* We're calling into EFI
* On exception entry from EL1, as part of stacking x18
* During CPU suspend
If we split the pointer in two (base, offset) then we could leave the base live in the thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.
The base and offset can live in the same cacheline and be loaded with ldp, so I don't see there being an access cost compared to a single variable.
Am I missing something (modulo us not agreeing on the utility of wiping the pointer)?
Will
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 5 lines · Will Deacon
Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the
kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?
I agree that allocating from a kmem_cache isn't ideal for safety. It's a compromise to reduce memory overhead.
Quoted 4 lines
The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't passVM_NO_GUARD. Looks like the same story for vmap stack :/
SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?
Quoted 5 lines
If we split the pointer in two (base, offset) then we could leave the base live in the
thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.
I sent out v13 with split pointers, but I'm not sure it's convenient to add an overflow check to kernel_ventry where the VMAP_STACK check is done. I suppose I could add a check to kernel_entry after we load x18 from tsk. Thoughts?
Sami
Will Deacon
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 9 lines · Sami Tolvanen
Quoted 5 lines · Will Deacon
Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the
kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?I agree that allocating from a
kmem_cacheisn't ideal for safety. It's a compromise to reduce memory overhead.
Do you think it would be a problem if we always allocated a page for the SCS?
Quoted 7 lines
Quoted 4 lines
The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't passVM_NO_GUARD. Looks like the same story for vmap stack :/SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?
Sorry, I'd got the SCS upside-down in my head (hey, that second 'S' stands for 'Stack'!). But I think I'm right about vmap stack, which feels a little fragile even though it seems to work out today with the very limited uses of VM_NO_GUARD.
Quoted 10 lines
Quoted 5 lines
If we split the pointer in two (base, offset) then we could leave the base live in the
thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.I sent out v13 with split pointers, but I'm not sure it's convenient to add an overflow check to
kernel_ventrywhere theVMAP_STACKcheck is done. I suppose I could add a check tokernel_entryafter we load x18 from tsk. Thoughts?
I'll take a look at v13, since at this stage I'm keen to get something queued up so that we can use it as a base for further improvements without you having to repost the whole stack every time.
Cheers,
Will
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 6 lines · Will Deacon
Quoted 2 lines · Sami Tolvanen
I agree that allocating from a
kmem_cacheisn't ideal for safety. It's a compromise to reduce memory overhead.Do you think it would be a problem if we always allocated a page for the SCS?
Yes, the memory overhead was deemed too large for Android devices, which have thousands of threads running.
Sami
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
In reply to Will DeaconQuoted 8 lines · Will Deacon
Quoted 7 lines · Sami Tolvanen
Quoted 5 lines · Will Deacon
Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the
kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?
Quoted 5 lines
Quoted 2 lines
I agree that allocating from a
kmem_cacheisn't ideal for safety. It's a compromise to reduce memory overhead.Do you think it would be a problem if we always allocated a page for the SCS?
I guess doing this safely and without wasting a page per task would only be possible in an elegant way once MTE lands on devices?
I wonder how bad context switch latency would be if the actual SCS was percpu and vmapped (starting at an offset inside the page such that the SCS can only grow up to something like 0x400 bytes before panicking the CPU) and the context switch path saved/restored the used part of the vmapped SCS into a smaller allocation from the slab allocator... presumably the SCS will usually just be something like one cacheline big? That probably only costs a moderate amount of time to copy...
Or as an extension of that, if the SCS copying turns out to be too costly, there could be a percpu LRU cache consisting of vmapped SCS pages, and whenever a task gets scheduled that doesn't have a vmapped SCS, it "swaps out" the contents of the least recently used vmapped SCS into the corresponding task's slab SCS, and "swaps in" from its own slab SCS into the vmapped SCS. And task migration would force "swapping out".
Not sure if this is a good idea, or if I'm just making things worse by suggesting extra complexity...
Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
In reply to Will DeaconQuoted 7 lines · Will Deacon
Quoted 6 lines · Sami Tolvanen
Quoted 4 lines · Will Deacon
The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't passVM_NO_GUARD. Looks like the same story for vmap stack :/
Quoted 7 lines
Quoted 2 lines
SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?
Sorry, I'd got the SCS upside-down in my head (hey, that second 'S' stands for 'Stack'!). But I think I'm right about vmap stack, which feels a little fragile even though it seems to work out today with the very limited uses of
VM_NO_GUARD.
Yeah, when VMAP_STACK was originally being developed, IIRC, there was an effort made to eliminate all the users of VM_NO_GUARD, and it looks like it's mostly there. Really the only use left is arm64's kernel image mapping routines, and then it's not actually used in the traditional sense -- it's just a boolean for whether to toss in a guard page at the end of the data section, and the VMAs are built manually. I think that code could actually be refactored to drop it too and then the only user would be KASAN, which, IIUC, wants to build consecutive vmap areas.
Signature · 6 lines
Kees Cook _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Steven Rostedt
maintainerRe: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)
In reply to Will DeaconQuoted 8 lines · On Mon, 20 Apr 2020 18:17:28 +0100 Will Deacon <[email protected]> wrote:
Makefile ifdef CONFIG_LIVEPATCH · line 873
Quoted 5 lines
Makefile +5
-0+ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif
CFLAGS_SCSwould seem more natural to me, although I see ftrace does it this way.
The CC_FLAGS_FTRACE was added by Heiko Carstens, and the "CC_FLAGS_" appears to be a common usage in s390 :-)
That said, I like the CC_FLAGS_ notation, because the Linux build system uses CFLAGS_* as commands:
CFLAGS_foo.o = x
CFLAGS_REMOVE_foo.o = y
And "CC_FLAGS_" is only for new flags and easy to search for.
-- Steve
PATCH v11 2/12 scs: add accounting
In reply to Sami TolvanenThis change adds accounting for the memory allocated for shadow stacks.
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 20 ++++++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid,sum_zone_node_page_state (nid,NR_KERNEL_SCS_BYTES ) /1024 , +#endif nid,K (sum_zone_node_page_state (nid,NR_PAGETABLE )), nid,K (node_page_state (pgdat,NR_UNSTABLE_NFS )), nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)show_val_kb (m,"SUnreclaim: " , sunreclaim);seq_printf (m,"KernelStack: %8lu kB\n" ,global_zone_page_state (NR_KERNEL_STACK_KB )); +#ifdef CONFIG_SHADOW_CALL_STACK +seq_printf (m,"ShadowCallStack:%8lu kB\n" , +global_zone_page_state (NR_KERNEL_SCS_BYTES ) /1024 ); +#endif show_val_kb (m,"PageTables: " ,global_zone_page_state (NR_PAGETABLE ));
include/linux/mmzone.h +3 -0
@@ -156,6 +156,9 @@ enum zone_stat_item {NR_MLOCK ,/* mlock()ed pages found and moved off LRU */ NR_PAGETABLE ,/* used for pagetables */ NR_KERNEL_STACK_KB ,/* measured in KiB */ +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +NR_KERNEL_SCS_BYTES ,/* measured in bytes */ +#endif/* Second 128 byte cacheline */ NR_BOUNCE , #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
@@ -12,6 +12,7 @@#include <linux/scs.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/vmstat.h> #include <asm/scs.h> static inline void *__scs_base (struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)vfree_atomic (s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return vmalloc_to_page (__scs_base (tsk)); +} +static int scs_cleanup (unsigned int cpu) {int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)kmem_cache_free (scs_cache, s); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return virt_to_page (__scs_base (tsk)); +} +void __init scs_init (void ) { scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)task_set_scs (tsk,__scs_base (tsk)); } +static void scs_account (struct task_struct *tsk,int account) +{ +mod_zone_page_state (page_zone (__scs_page (tsk)),NR_KERNEL_SCS_BYTES , + account *SCS_SIZE ); +} +int scs_prepare (struct task_struct *tsk,int node) {void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)return -ENOMEM ;task_set_scs (tsk, s); +scs_account (tsk,1 ); +return 0 ; }
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)WARN_ON (scs_corrupted (tsk)); +scs_account (tsk,-1 );task_set_scs (tsk,NULL );scs_free (s); }
mm/page_alloc.c +6 -0
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " managed:%lukB" " mlocked:%lukB" " kernel_stack:%lukB" +#ifdef CONFIG_SHADOW_CALL_STACK + " shadow_call_stack:%lukB" +#endif" pagetables:%lukB" " bounce:%lukB" " free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)K (zone_managed_pages (zone)),K (zone_page_state (zone,NR_MLOCK )),zone_page_state (zone,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK +zone_page_state (zone,NR_KERNEL_SCS_BYTES ) /1024 , +#endifK (zone_page_state (zone,NR_PAGETABLE )),K (zone_page_state (zone,NR_BOUNCE )), K(free_pcp),
mm/vmstat.c +3 -0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {"nr_mlock" ,"nr_page_table_pages" ,"nr_kernel_stack" , +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +"nr_shadow_call_stack_bytes" , +#endif"nr_bounce" , #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages",
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Will Deacon
maintainerRe: [PATCH v11 02/12] scs: add accounting
Quoted 28 lines · Sami Tolvanen
This change adds accounting for the memory allocated for shadow stacks.
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 20 ++++++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 42 insertions(+)drivers/base/node.c +3
-0@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
drivers/base/node.c static ssize_t node_read_meminfo(struct device *dev, · line 446
Quoted 6 lines
drivers/base/node.c +3
-0@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024, +#endif
Why not just use KB everywhere instead of repeated division by 1024?
Will
Re: [PATCH v11 02/12] scs: add accounting
Quoted 5 lines · Will Deacon
drivers/base/node.c static ssize_t node_read_meminfo(struct device *dev, · line 446
Quoted 3 lines
drivers/base/node.c +3
-0+#ifdef CONFIG_SHADOW_CALL_STACK + nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024, +#endif Why not just use KB everywhere instead of repeated division by 1024?
This was to correctly calculate memory usage with shadow stacks <1024 bytes. I don't think we need that anymore, so I'll change this to _KB in the next version.
Sami
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH v11 3/12 scs: add support for stack usage debugging
In reply to Sami TolvanenImplements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.
--- kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static inline unsigned long scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static DEFINE_SPINLOCK (lock); +static unsigned long highest; +unsigned long used =scs_used (tsk); + +if (used <= highest) +return ; + +spin_lock (&lock); + +if (used > highest) { +pr_info ("%s (%d): highest shadow stack usage: %lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); + highest = used; + } + +spin_unlock (&lock); +} +#else +static inline void scs_check_usage (struct task_struct *tsk) +{ +} +#endif +bool scs_corrupted (struct task_struct *tsk) {unsigned long *magic =scs_magic (__scs_base (tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)return ;WARN_ON (scs_corrupted (tsk)); +scs_check_usage (tsk);scs_account (tsk,-1 );task_set_scs (tsk,NULL );
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Will Deacon
maintainerRe: [PATCH v11 03/12] scs: add support for stack usage debugging
Quoted 16 lines · Sami Tolvanen
Implements
CONFIG_DEBUG_STACK_USAGEfor shadow stacks. When enabled, also prints out the highest shadow stack usage per process.Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>--- kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+)kernel/scs.c +0
-0@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; }
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 195
Quoted 10 lines
kernel/scs.c +9
-0+#ifdef CONFIG_DEBUG_STACK_USAGE +static inline unsigned long scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++;
I think the expectation is that the caller has already checked that the stack is not corrupted, so I'd probably throw a couple of underscores in front of the function name, along with a comment.
Also, is tsk ever != current?
Quoted 12 lines
kernel/scs.c +12
-0+ +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static DEFINE_SPINLOCK (lock); +static unsigned long highest; +unsigned long used =scs_used (tsk); + +if (used <= highest) +return ;
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 217
Quoted 10 lines
kernel/scs.c +10
-0+ +spin_lock (&lock); + +if (used > highest) { +pr_info ("%s (%d): highest shadow stack usage: %lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); + highest = used; + } + +spin_unlock (&lock);
Do you really need this lock? I'd have thought you could cmpxchg() highest instead.
Will
Re: [PATCH v11 03/12] scs: add support for stack usage debugging
Quoted 5 lines · Will Deacon
Quoted 5 lines
kernel/scs.c +5
-0+#ifdef CONFIG_DEBUG_STACK_USAGE +static inline unsigned long scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =scs_magic (p);
Quoted 8 lines
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 195
Quoted 4 lines
kernel/scs.c +4
-0+unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++;I think the expectation is that the caller has already checked that the stack is not corrupted, so I'd probably throw a couple of underscores in front of the function name, along with a comment.
Correct. I'll do that.
Quoted 1 line
Also, is tsk ever != current?
This is only called from scs_release(), so tsk is never current.
Quoted 13 lines
Quoted 13 lines
kernel/scs.c +13
-0+static void scs_check_usage (struct task_struct *tsk) +{ +static DEFINE_SPINLOCK (lock); +static unsigned long highest; +unsigned long used =scs_used (tsk); + +if (used <= highest) +return ; + +spin_lock (&lock); + +if (used > highest) { + pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
Quoted 8 lines
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 217
Quoted 5 lines
kernel/scs.c +5
-0+ tsk->comm, task_pid_nr(tsk), used); + highest = used; + } + +spin_unlock (&lock);Do you really need this lock? I'd have thought you could
cmpxchg()highest instead.
This is similar to check_stack_usage in kernel/exit.c, but yes, I can change this to a cmpxchg() loop instead.
Sami
PATCH v11 4/12 scs: disable when function graph tracing is enabled
In reply to Sami TolvanenThe graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Peter Zijlstra
maintainerRe: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
Quoted 15 lines · Sami Tolvanen
The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for
DYNAMIC_FTRACE_WITH_REGSmodifies the LR before it is saved to the shadow stack, and is compatible.
Quoted 8 lines
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
SCS would actually provide another way to do return hooking. An arguably much saner model at that.
The 'normal' way is to (temporary) replace the on-stack return value, and then replace it back in the return handler. This is because we can't simply push a fake return on the stack, because that would wreck the expected stack layout of the regular function.
But there is nothing that would stop us from pushing an extra entry on the SCS. It would in fact be a much cleaner solution. The entry hook sticks an extra entry on the SCS, the function ignores what's on the normal stack and pops from the SCS, we return to the exit handler, which in turn pops from the SCS stack at which point we're back to regular.
The only 'funny' is that the exit handler itself should not push to the SCS, or we should frob the return-to-exit-handler such that it lands after the push.
Quoted 12 lines
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>Reviewed-by Mark Rutland <[email protected]>--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)arch/Kconfig +0
-0
arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548
Quoted 7 lines
arch/Kconfig +1
-0@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses a
AFAICT you also need to kill KRETPROBES, which plays similar games. And doesn't BPF also do stuff like this?
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Mark Rutland
reviewerRe: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
Hi Peter,
Quoted 33 lines · Peter Zijlstra
Quoted 15 lines · Sami Tolvanen
The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for
DYNAMIC_FTRACE_WITH_REGSmodifies the LR before it is saved to the shadow stack, and is compatible.Quoted 8 lines
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
SCS would actually provide another way to do return hooking. An arguably much saner model at that.
The 'normal' way is to (temporary) replace the on-stack return value, and then replace it back in the return handler. This is because we can't simply push a fake return on the stack, because that would wreck the expected stack layout of the regular function.
Quoted 5 lines
But there is nothing that would stop us from pushing an extra entry on the SCS. It would in fact be a much cleaner solution. The entry hook sticks an extra entry on the SCS, the function ignores what's on the normal stack and pops from the SCS, we return to the exit handler, which in turn pops from the SCS stack at which point we're back to regular.
For background: on arm64 we wanted to use DYNAMIC_FTRACE_WITH_REGS since we already have to use that to handle pointer authentication, and didn't want to gain more ways of implementing ftrace.
Arguably we should move the dependency into the arm64 Kconfig for ARCH_SUPPORTS_SHADOW_CALL_STACK.
Quoted 17 lines
The only 'funny' is that the exit handler itself should not push to the SCS, or we should frob the return-to-exit-handler such that it lands after the push.
Quoted 13 lines
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>Reviewed-by Mark Rutland <[email protected]>--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)arch/Kconfig +0
-0@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
Quoted 8 lines
arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548
Quoted 6 lines
arch/Kconfig +1
-0config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses aAFAICT you also need to kill KRETPROBES, which plays similar games.
Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...
Quoted 1 line
And doesn't BPF also do stuff like this?
Can BPF mess with return addresses now!?
Thanks,
Mark.
Peter Zijlstra
maintainerRe: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
Quoted 15 lines · Mark Rutland
Quoted 6 lines
Quoted 6 lines
arch/Kconfig +0
-0@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACKQuoted 8 lines
arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548
Quoted 6 lines
arch/Kconfig +1
-0config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses aAFAICT you also need to kill KRETPROBES, which plays similar games.
Quoted 5 lines
Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...
To be fair, I've not looked at the arm64 implementation. x86 does gross things like ftrace does. On x86 ftrace_graph and kretprobe also can't be on at the same time for the same function, there's some yuck around there.
Rostedt was recently talking about cleaning some of that up.
But if kretprobe can work on arm64, then ftrace_graph can too, but I think that links back to what you said earlier, you didn't want more ftrace variants or something.
Quoted 3 lines
Quoted 1 line
And doesn't BPF also do stuff like this?
Can BPF mess with return addresses now!?
At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Mark Rutland
reviewerRe: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
Quoted 22 lines · Peter Zijlstra
Quoted 15 lines · Mark Rutland
Quoted 6 lines
Quoted 6 lines
arch/Kconfig +0
-0@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACKQuoted 8 lines
arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548
Quoted 6 lines
arch/Kconfig +1
-0config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER depends on ARCH_SUPPORTS_SHADOW_CALL_STACK help This option enables Clang's Shadow Call Stack, which uses aAFAICT you also need to kill KRETPROBES, which plays similar games.
Quoted 5 lines
Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...
Quoted 4 lines
To be fair, I've not looked at the arm64 implementation. x86 does gross things like ftrace does. On x86
ftrace_graphand kretprobe also can't be on at the same time for the same function, there's some yuck around there.
I can imagine the same holds true for us there.
Quoted 5 lines
Rostedt was recently talking about cleaning some of that up.
But if kretprobe can work on arm64, then
ftrace_graphcan too, but I think that links back to what you said earlier, you didn't want more ftrace variants or something.
I just want to avoid yet another implementation of the underlying mechanism. For DYNAMIC_FTRACE_WITH_REGS we can mess with the LR before pauth or SCS sees it, so those definitely work.
If KRETPROBES works by messing with the LR at the instnat the function is entered, that should work similarly. If it works by replacing the RET it should also work out since any pauth/SCS work will have been undone by that point. If it attempts to mess with the return address in the middle of a function then it's not reliable today.
I'll take a look, since
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
And doesn't BPF also do stuff like this?
Can BPF mess with return addresses now!?
At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.
Sounds like I might need to do some digging...
Mark.
Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
Quoted 5 lines · Mark Rutland
If KRETPROBES works by messing with the LR at the instnat the function is entered, that should work similarly. If it works by replacing the RET it should also work out since any pauth/SCS work will have been undone by that point. If it attempts to mess with the return address in the middle of a function then it's not reliable today.
I did initially have a patch to disable kretprobes (until v5), but as Mark pointed out back then, the return address is modified before it gets pushed to the shadow stack, so there was no conflict with SCS. I confirmed this on arm64, but haven't looked at other architectures.
Sami
Steven Rostedt
maintainerRe: [PATCH v11 04/12] scs: disable when function graph tracing is enabled
In reply to Mark RutlandQuoted 8 lines · On Fri, 17 Apr 2020 16:46:14 +0100 Mark Rutland <[email protected]> wrote:
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
And doesn't BPF also do stuff like this?
Can BPF mess with return addresses now!?
At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.
Sounds like I might need to do some digging...
May want to ping Alexei. It appears that if BPF adds a direct hook to a function, it will prevent the function graph tracer from tracing it. :-/
-- Steve
PATCH v11 5/12 arm64: reserve x18 from general allocation with SCS
In reply to Sami TolvanenReserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.
--- arch/arm64/Makefile | 4 ++++ 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
@@ -81,6 +81,10 @@ endif KBUILD_CFLAGS += $(branch-prot-flags-y) +ifeq ($(CONFIG_SHADOW_CALL_STACK), y) +KBUILD_CFLAGS += -ffixed-x18 +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 6/12 arm64: preserve x18 when CPU is suspended
In reply to Sami TolvanenDon't lose the current task's shadow stack when the CPU is suspended.
--- arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/mm/proc.S | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@#ifndef __ASM_SUSPEND_H#define __ASM_SUSPEND_H -#define NR_CTX_REGS 12 +#define NR_CTX_REGS 13#define NR_CALLEE_SAVED_REGS 12/*
arch/arm64/mm/proc.S +14 -0
@@ -58,6 +58,8 @@ * cpu_do_suspend - save CPU registers context * * x0: virtual address of context pointer + * + * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. */ SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif stp x8, x9, [x0, #48] stp x10, x11, [x0, #64] stp x12, x13, [x0, #80] + /* + * Save x18 as it may be used as a platform register, e.g. by shadow + * call stack. + */ + str x18, [x0, #96] ret SYM_FUNC_END(cpu_do_suspend)
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + /* + * Restore x18, as it may be used as a platform register, and clear + * the buffer to minimize the risk of exposure when used for shadow + * call stack. + */ + ldr x18, [x0, #96] + str xzr, [x0, #96] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 7/12 arm64: efi: restore x18 if it was corrupted
In reply to Sami TolvanenIf we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.
--- arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper) ldp x29, x30, [sp], #32 b.ne 0f ret -0: b efi_handle_corrupted_x18 // tail call +0: + /* + * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a + * shadow stack pointer, which we need to restore before returning to + * potentially instrumented code. This is safe because the wrapper is + * called with preemption disabled and a separate shadow stack is used + * for interrupts. + */ + mov x18, x2 + b efi_handle_corrupted_x18 // tail call ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 8/12 arm64: vdso: disable Shadow Call Stack
In reply to Sami TolvanenShadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.
--- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING VDSO_LDFLAGS := -Bsymbolic -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) KBUILD_CFLAGS += $(DISABLE_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 9/12 arm64: disable SCS for hypervisor code
In reply to Sami TolvanenDisable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.
--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 10/12 arm64: implement Shadow Call Stack
In reply to Sami TolvanenThis change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
--- arch/arm64/Kconfig | 5 ++++ arch/arm64/include/asm/scs.h | 37 +++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 3 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 33 +++++++++++++++++++++-- arch/arm64/kernel/head.S | 8 ++++++ arch/arm64/kernel/irq.c | 2 ++ arch/arm64/kernel/process.c | 2 ++ arch/arm64/kernel/scs.c | 40 ++++++++++++++++++++++++++++ arch/arm64/kernel/smp.c | 4 +++ 11 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
@@ -64,6 +64,7 @@ config ARM64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 +# Supported by clang >= 7.0 +config CC_HAVE_SHADOW_CALL_STACK + def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help---
arch/arm64/include/asm/scs.h new file +37 -0
@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifndef __ASSEMBLY__ + +#include <linux/scs.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +extern void scs_init_irq (void ); + +static __always_inline void scs_save(struct task_struct *tsk) +{ +void *s; + + asmvolatile ("mov %0, x18" :"=r" (s)); +task_set_scs (tsk, s); +} + +static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" ); +} + +#else /* CONFIG_SHADOW_CALL_STACK */ + +static inline void scs_init_irq (void ) {} +static inline void scs_save (struct task_struct *tsk) {} +static inline void scs_overflow_check (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* __ASSEMBLY __ */ + +#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
@@ -41,6 +41,9 @@ struct thread_info {#endif } preempt; }; +#ifdef CONFIG_SHADOW_CALL_STACK +void *shadow_call_stack; +#endif };#define thread_saved_pc (tsk) \
arch/arm64/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_SSBD) += ssbd.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-y += vdso/ probes/ obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
@@ -33,6 +33,9 @@ int main(void) DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));#ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif +#ifdef CONFIG_SHADOW_CALL_STACK + DEFINE(TSK_TI_SCS, offsetof(struct task_struct, thread_info.shadow_call_stack));#endif DEFINE (TSK_STACK , offsetof(struct task_struct , stack));#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
@@ -179,6 +179,11 @@ alternative_cb_end apply_ssbd 1, x22, x23 ptrauth_keys_install_kernel tsk, 1, x20, x22, x23 + +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [tsk, #TSK_TI_SCS] // Restore shadow call stack + str xzr, [tsk, #TSK_TI_SCS] // Limit visibility of saved SCS +#endif .else add x21, sp, #S_FRAME_SIZE get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif ct_user_enter .endif +#ifdef CONFIG_SHADOW_CALL_STACK + .if \el == 0 + str x18, [tsk, #TSK_TI_SCS] // Save shadow call stack + .endif +#endif + #ifdef CONFIG_ARM64_SW_TTBR0_PAN /* * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .macro irq_stack_entry mov x19, sp // preserve the original sp +#ifdef CONFIG_SHADOW_CALL_STACK + mov x24, x18 // preserve the original shadow stack +#endif /* * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 /* switch to the irq stack */ mov sp, x26 + +#ifdef CONFIG_SHADOW_CALL_STACK + /* also switch to the irq shadow stack */ + ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26 +#endif + 9998: .endm /* - * x19 should be preserved between irq_stack_entry and - * irq_stack_exit. + * The callee-saved regs (x19-x29) should be preserved between + * irq_stack_entry and irq_stack_exit, but note that kernel_entry + * uses x20-x23 to store data for later use. */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_SHADOW_CALL_STACK + mov x18, x24 +#endif .endm /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to) mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, 1, x8, x9, x10 +#ifdef CONFIG_SHADOW_CALL_STACK + str x18, [x0, #TSK_TI_SCS] + ldr x18, [x1, #TSK_TI_SCS] + str xzr, [x1, #TSK_TI_SCS] // limit visibility of saved SCS +#endif ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) stp xzr, x30, [sp, #-16]! mov x29, sp +#ifdef CONFIG_SHADOW_CALL_STACK + adr_l x18, init_shadow_call_stack // Set shadow call stack +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched) ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow msr sp_el0, x2 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [x2, #TSK_TI_SCS] // set shadow call stack + str xzr, [x2, #TSK_TI_SCS] // limit visibility of saved SCS +#endif mov x29, #0 mov x30, #0 b secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
@@ -21,6 +21,7 @@#include <linux/vmalloc.h> #include <asm/daifflags.h> #include <asm/vmap_stack.h> +#include <asm/scs.h> unsigned long irq_err_count;
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)void __init init_IRQ (void ) {init_irq_stacks (); +scs_init_irq ();irqchip_init ();if (!handle_arch_irq)panic ("No interrupt controller found." );
arch/arm64/kernel/process.c +2 -0
@@ -52,6 +52,7 @@#include <asm/mmu_context.h> #include <asm/processor.h> #include <asm/pointer_auth.h> +#include <asm/scs.h> #include <asm/stacktrace.h> #if defined(CONFIG_STACKPROTECTOR ) && !defined(CONFIG_STACKPROTECTOR_PER_TASK )
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,entry_task_switch (next);uao_thread_switch (next);ssbs_thread_switch (next); +scs_overflow_check (next);/* * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
@@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/percpu.h> +#include <linux/vmalloc.h> +#include <asm/pgtable.h> +#include <asm/scs.h> + +DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); + +#ifndef CONFIG_SHADOW_CALL_STACK_VMAP +DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack) +__aligned(SCS_SIZE) ; +#endif + +void scs_init_irq (void ) +{ +int cpu; + +for_each_possible_cpu (cpu) { +#ifdef CONFIG_SHADOW_CALL_STACK_VMAP +unsigned long *p; + + p =__vmalloc_node_range (PAGE_SIZE ,SCS_SIZE , +VMALLOC_START ,VMALLOC_END , +GFP_SCS ,PAGE_KERNEL , +0 ,cpu_to_node (cpu), +__builtin_return_address (0 )); + +per_cpu (irq_shadow_call_stack_ptr, cpu) = p; +#else +per_cpu (irq_shadow_call_stack_ptr, cpu) = +per_cpu (irq_shadow_call_stack, cpu); +#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */ + } +}
arch/arm64/kernel/smp.c +4 -0
@@ -46,6 +46,7 @@#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/processor.h> +#include <asm/scs.h> #include <asm/smp_plat.h> #include <asm/sections.h> #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)unsigned int cpu =smp_processor_id ();const struct cpu_operations *ops =get_cpu_ops (cpu); +/* Save the shadow stack pointer before exiting the idle task */ +scs_save (current); +idle_task_exit ();local_daif_mask ();
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 11/12 arm64: scs: add shadow stacks for SDEI
In reply to Sami TolvanenThis change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.
--- arch/arm64/include/asm/scs.h | 2 + arch/arm64/kernel/entry.S | 14 ++++- arch/arm64/kernel/scs.c | 106 +++++++++++++++++++++++++++++------ arch/arm64/kernel/sdei.c | 7 +++ 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
@@ -9,6 +9,7 @@#ifdef CONFIG_SHADOW_CALL_STACK extern void scs_init_irq (void ); +extern int scs_init_sdei (void ); static __always_inline void scs_save(struct task_struct *tsk) {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)#else /* CONFIG_SHADOW_CALL_STACK */ static inline void scs_init_irq (void ) {} +static inline int scs_init_sdei (void ) {return 0 ; }static inline void scs_save (struct task_struct *tsk) {}static inline void scs_overflow_check (struct task_struct *tsk) {}
arch/arm64/kernel/entry.S +13 -1
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler) mov x19, x1 +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] +#endif + #ifdef CONFIG_VMAP_STACK /* * entry.S may have been using sp as a scratch register, find whether * this is a normal or critical event and switch to the appropriate * stack for this CPU. */ - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] cbnz w4, 1f ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 b 2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler) mov sp, x5 #endif +#ifdef CONFIG_SHADOW_CALL_STACK + /* Use a separate shadow call stack for normal and critical events */ + cbnz w4, 3f + ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 + b 4f +3: ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 +4: +#endif + /* * We may have interrupted userspace, or a guest, or exit-from or * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16
121 lines not on this page · raw message
arch/arm64/kernel/sdei.c +7 -0
@@ -13,6 +13,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/ptrace.h> +#include <asm/scs.h> #include <asm/sections.h> #include <asm/stacktrace.h> #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)return 0 ; } +if (scs_init_sdei ()) { +if (IS_ENABLED (CONFIG_VMAP_STACK )) +free_sdei_stacks (); +return 0 ; + } + sdei_exit_mode = (conduit ==SMCCC_CONDUIT_HVC ) ?SDEI_EXIT_HVC :SDEI_EXIT_SMC ;#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v11 12/12 efi/libstub: disable SCS
In reply to Sami TolvanenShadow stacks are not available in the EFI stub, filter out SCS flags.
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
v12 cover add support for Clang's Shadow Call Stack
In reply to Sami TolvanenThis patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Changes in v12:
- Removed CONFIG_SHADOW_CALL_STACK_VMAP.
- Added CC_IS_CLANG as a dependency to CONFIG_SHADOW_CALL_STACK.
- Changed SCS_END_MAGIC to use POISON_POINTER_DELTA.
- Removed the unnecessary scs_set_magic() helper function.
- Moved scs_task_reset() and scs_corrupted() to scs.h, along with __scs_magic() and __scs_base().
- Removed a redundant warning from memory allocation.
- Removed an unnecessary task_set_scs() call from scs_release().
- Changed the accounting code to calculate KiB instead of bytes.
- Replaced the lock in scs_check_usage() with a cmpxchg() loop.
Changes in v11:
- Rebased, added maintainers for kernel/ changes.
Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.
Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.
Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.
Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.
Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler
Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information
Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical
Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit
Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions
Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS
Makefile | 6 ++ arch/Kconfig | 26 ++++++ arch/arm64/Kconfig | 5 ++ arch/arm64/Makefile | 4 + arch/arm64/include/asm/kvm_hyp.h | 2 +- arch/arm64/include/asm/scs.h | 34 ++++++++ arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/include/asm/thread_info.h | 3 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 + arch/arm64/kernel/efi-rt-wrapper.S | 11 ++- arch/arm64/kernel/entry.S | 47 +++++++++- arch/arm64/kernel/head.S | 8 ++ arch/arm64/kernel/process.c | 2 + arch/arm64/kernel/scs.c | 21 +++++ arch/arm64/kernel/smp.c | 4 + arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/mm/proc.S | 14 +++ drivers/base/node.c | 6 ++ drivers/firmware/efi/libstub/Makefile | 3 + fs/proc/meminfo.c | 4 + include/linux/compiler-clang.h | 6 ++ include/linux/compiler_types.h | 4 + include/linux/mmzone.h | 3 + include/linux/scs.h | 92 ++++++++++++++++++++ init/init_task.c | 8 ++ kernel/Makefile | 1 + kernel/fork.c | 9 ++ kernel/sched/core.c | 2 + kernel/scs.c | 121 ++++++++++++++++++++++++++ mm/page_alloc.c | 6 ++ mm/vmstat.c | 3 + 32 files changed, 456 insertions(+), 7 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
base-commit: ae83d0b416db002fe95601e7f97f64b59514d936
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 1/12 add support for Clang's Shadow Call Stack (SCS)
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
--- Makefile | 6 +++ arch/Kconfig | 25 +++++++++ include/linux/compiler-clang.h | 6 +++ include/linux/compiler_types.h | 4 ++ include/linux/scs.h | 92 ++++++++++++++++++++++++++++++++++ init/init_task.c | 8 +++ kernel/Makefile | 1 + kernel/fork.c | 9 ++++ kernel/sched/core.c | 2 + kernel/scs.c | 67 +++++++++++++++++++++++++ 10 files changed, 220 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
Makefile +6 -0
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
arch/Kconfig +25 -0
@@ -533,6 +533,31 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use. + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help
include/linux/compiler-clang.h +6 -0
@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif
include/linux/compiler_types.h +4 -0
@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif
include/linux/scs.h new file +92 -0
92 lines not on this page · raw message
init/init_task.c +8 -0
@@ -11,6 +11,7 @@#include <linux/mm.h> #include <linux/audit.h> #include <linux/numa.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task };EXPORT_SYMBOL (init_task); +#ifdef CONFIG_SHADOW_CALL_STACK +unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data +__aligned(SCS_SIZE) = { + [(SCS_SIZE /sizeof (long )) -1 ] =SCS_END_MAGIC +}; +#endif +/* * Initial thread structure. Alignment of this is handled by a special * linker map entry.
kernel/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/fork.c +9 -0
@@ -94,6 +94,7 @@#include <linux/thread_info.h> #include <linux/stackleak.h> #include <linux/kasan.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)void free_task (struct task_struct *tsk) { +scs_release (tsk); +#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void) NULL, free_vm_stack_cache);#endif +scs_init (); +lockdep_init_task (&init_task);uprobes_init (); }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)if (err) goto free_stack; + err =scs_prepare (tsk, node); +if (err) + goto free_stack; +#ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP
kernel/scs.c new file +67 -0
67 lines not on this page · raw message
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 371 lines · Sami Tolvanen
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>Reviewed-by Miguel Ojeda <[email protected]>--- Makefile | 6 +++ arch/Kconfig | 25 +++++++++ include/linux/compiler-clang.h | 6 +++ include/linux/compiler_types.h | 4 ++ include/linux/scs.h | 92 ++++++++++++++++++++++++++++++++++ init/init_task.c | 8 +++ kernel/Makefile | 1 + kernel/fork.c | 9 ++++ kernel/sched/core.c | 2 + kernel/scs.c | 67 +++++++++++++++++++++++++ 10 files changed, 220 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.cMakefile +6
-0@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)arch/Kconfig +25
-0@@ -533,6 +533,31 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the ones + documented for user space. The kernel must store addresses of shadow + stacks used by other tasks and interrupt handlers in memory, which + means an attacker capable of reading and writing arbitrary memory + may be able to locate them and hijack control flow by modifying + shadow stacks that are not currently in use. + config HAVE_ARCH_WITHIN_STACK_FRAMES bool helpinclude/linux/compiler-clang.h +6
-0@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif include/linux/compiler_types.h +4
-0@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif include/linux/scs.h new file +92
-0@@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#ifndef _LINUX_SCS_H +#define _LINUX_SCS_H + +#include <linux/gfp.h> +#include <linux/poison.h> +#include <linux/sched.h> +#include <asm/page.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +/* +* In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit +* architecture) provided ~40% safety margin on stack usage while keeping +* memory allocation overhead reasonable. +*/ +#define SCS_SIZE 1024UL +#define GFP_SCS (GFP_KERNEL | __GFP_ZERO) + +/* An illegal pointer value to mark the end of the shadow stack. */ +#define SCS_END_MAGIC (0x5f6UL + POISON_POINTER_DELTA) + +#define task_scs (tsk) (task_thread_info(tsk)->shadow_call_stack) + +static inline void task_set_scs (struct task_struct *tsk,void *s) +{ +task_scs (tsk) = s; +} + +extern void scs_init (void ); + +static inline void *__scs_base (struct task_struct *tsk) +{ +/* +* To minimize the risk of exposure, architectures may clear a +* task's thread_info::shadow_call_stack while that task is +* running, and only save/restore the active shadow call stack +* pointer when the usual register may be clobbered (e.g. across +* context switches). +* +* The shadow call stack is aligned to SCS_SIZE, and grows +* upwards, so we can mask out the low bits to extract the base +* when the task is not running. +*/ +return (void *)((unsigned long )task_scs (tsk) & ~(SCS_SIZE -1 )); +} + +static inline void scs_task_reset (struct task_struct *tsk) +{ +/* +* Reset the shadow stack to the base address in case the task +* is reused. +*/ +task_set_scs (tsk,__scs_base (tsk)); +} + +extern int scs_prepare (struct task_struct *tsk,int node); + +static inline unsigned long *__scs_magic (void *s) +{ +return (unsigned long *)(s +SCS_SIZE ) -1 ; +} + +static inline bool scs_corrupted (struct task_struct *tsk) +{ +unsigned long *magic =__scs_magic (__scs_base (tsk)); + +return READ_ONCE_NOCHECK (*magic) !=SCS_END_MAGIC ; +} + +extern void scs_release (struct task_struct *tsk); + +#else /* CONFIG_SHADOW_CALL_STACK */ + +#define task_scs (tsk) NULL + +static inline void task_set_scs (struct task_struct *tsk,void *s) {} +static inline void scs_init (void ) {} +static inline void scs_task_reset (struct task_struct *tsk) {} +static inline int scs_prepare (struct task_struct *tsk,int node) {return 0 ; } +static inline bool scs_corrupted (struct task_struct *tsk) {return false; } +static inline void scs_release (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* _LINUX_SCS_H */ init/init_task.c +8
-0@@ -11,6 +11,7 @@#include <linux/mm.h> #include <linux/audit.h> #include <linux/numa.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <linux/uaccess.h> @@ -185,6 +186,13 @@ struct task_struct init_task };EXPORT_SYMBOL (init_task); +#ifdef CONFIG_SHADOW_CALL_STACK +unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data +__aligned(SCS_SIZE) = { + [(SCS_SIZE /sizeof (long )) -1 ] =SCS_END_MAGIC +}; +#endif +/* * Initial thread structure. Alignment of this is handled by a special * linker map entry. kernel/Makefile +1
-0@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/kernel/fork.c +9
-0@@ -94,6 +94,7 @@#include <linux/thread_info.h> #include <linux/stackleak.h> #include <linux/kasan.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> @@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)void free_task (struct task_struct *tsk) { +scs_release (tsk); +#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info, @@ -840,6 +843,8 @@ void __init fork_init(void) NULL, free_vm_stack_cache);#endif +scs_init (); +lockdep_init_task (&init_task);uprobes_init (); }@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)if (err) goto free_stack; + err =scs_prepare (tsk, node); +if (err) + goto free_stack; +#ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under kernel/sched/core.c +2
-0@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h> @@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP kernel/scs.c new file +55
-0@@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/kasan.h> +#include <linux/scs.h> +#include <linux/slab.h> +#include <asm/scs.h> + +static struct kmem_cache *scs_cache; + +static void *scs_alloc (int node) +{ +void *s; + + s =kmem_cache_alloc_node (scs_cache,GFP_SCS , node); +if (s) { + *__scs_magic (s) =SCS_END_MAGIC ; +/* +* Poison the allocation to catch unintentional accesses to +* the shadow stack when KASAN is enabled. +*/ +kasan_poison_object_data (scs_cache, s); + } + +return s; +} + +static void scs_free (void *s) +{ +kasan_unpoison_object_data (scs_cache, s); +kmem_cache_free (scs_cache, s); +} + +void __init scs_init (void ) +{ + scs_cache =kmem_cache_create ("scs_cache" ,SCS_SIZE ,SCS_SIZE , +0 ,NULL ); +} + +int scs_prepare (struct task_struct *tsk,int node) +{ +void *s; + + s =scs_alloc (node); +if (!s) +return -ENOMEM ; + +task_set_scs (tsk, s); +return 0 ; +} +
kernel/scs.c line 65
Quoted 10 lines
kernel/scs.c new file +10
-0+void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s) +return ; + +WARN_ON (scs_corrupted (tsk)); +
I'd like to have task_set_scs(tsk, NULL) retained here, to avoid need to depend on the released task memory getting scrubbed at a later time. Exposures of this pointer would expose the region of other SCSs.
-Kees
Quoted 5 lines
kernel/scs.c new file +2
-0+scs_free (s); +}Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Signature · 1 line
Kees Cook
Will Deacon
maintainerRe: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 14 lines · Kees Cook
kernel/scs.c line 65
Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10
-0+void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s) +return ; + +WARN_ON (scs_corrupted (tsk)); +I'd like to have
task_set_scs(tsk, NULL)retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.
Hmm, doesn't it get zeroed almost immediately by kmem_cache_free() if INIT_ON_FREE_DEFAULT_ON is set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in the task_struct and treating this specially feels odd to me.
Will
Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 16 lines · Will Deacon
Quoted 14 lines · Kees Cook
kernel/scs.c line 65
Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10
-0+void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s) +return ; + +WARN_ON (scs_corrupted (tsk)); +I'd like to have
task_set_scs(tsk, NULL)retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.
Quoted 4 lines
Hmm, doesn't it get zeroed almost immediately by
kmem_cache_free()ifINIT_ON_FREE_DEFAULT_ONis set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in thetask_structand treating this specially feels odd to me.
That's going to be an uncommon config except for the most paranoid of system builders. :) Having this get wiped particular thing wiped is just a decent best practice for what is otherwise treated as a "secret", just like crypto routines wipe their secrets before free().
Signature · 1 line
Kees Cook
Will Deacon
maintainerRe: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 17 lines · Kees Cook
Quoted 16 lines · Will Deacon
Quoted 14 lines · Kees Cook
kernel/scs.c line 65
Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10
-0+void scs_release (struct task_struct *tsk) +{ +void *s; + + s =__scs_base (tsk); +if (!s) +return ; + +WARN_ON (scs_corrupted (tsk)); +I'd like to have
task_set_scs(tsk, NULL)retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.
Quoted 7 lines
Quoted 4 lines
Hmm, doesn't it get zeroed almost immediately by
kmem_cache_free()ifINIT_ON_FREE_DEFAULT_ONis set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in thetask_structand treating this specially feels odd to me.That's going to be an uncommon config except for the most paranoid of system builders. :)
Sounds like a perfect fit, then ;)
Quoted 3 lines
Having this get wiped particular thing wiped is just
a decent best practice for what is otherwise treated as a "secret", just like crypto routines wipe their secrets beforefree().
Sorry, but I don't buy that analogy. The SCS pointer is stored in memory all over the place and if it needs to treated in the same way as crypto secrets then this whole thing needs rethinking. On top of that, where crypto routines may wipe their secrets, we don't do what is being proposed for the SCS pointer to other similar pieces of data, such as pointer authentication keys.
Will
PATCH v12 2/12 scs: add accounting
In reply to Sami TolvanenThis change adds accounting for the memory allocated for shadow stacks.
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 16 ++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 38 insertions(+)
drivers/base/node.c +6 -0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid,sum_zone_node_page_state (nid,NR_KERNEL_SCS_KB ), +#endif nid,K (sum_zone_node_page_state (nid,NR_PAGETABLE )), nid,K (node_page_state (pgdat,NR_UNSTABLE_NFS )), nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)show_val_kb (m,"SUnreclaim: " , sunreclaim);seq_printf (m,"KernelStack: %8lu kB\n" ,global_zone_page_state (NR_KERNEL_STACK_KB )); +#ifdef CONFIG_SHADOW_CALL_STACK +seq_printf (m,"ShadowCallStack:%8lu kB\n" , +global_zone_page_state (NR_KERNEL_SCS_KB )); +#endif show_val_kb (m,"PageTables: " ,global_zone_page_state (NR_PAGETABLE ));
include/linux/mmzone.h +3 -0
@@ -156,6 +156,9 @@ enum zone_stat_item {NR_MLOCK ,/* mlock()ed pages found and moved off LRU */ NR_PAGETABLE ,/* used for pagetables */ NR_KERNEL_STACK_KB ,/* measured in KiB */ +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +NR_KERNEL_SCS_KB ,/* measured in KiB */ +#endif/* Second 128 byte cacheline */ NR_BOUNCE , #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +16 -0
@@ -6,8 +6,10 @@*/ #include <linux/kasan.h> +#include <linux/mm.h> #include <linux/scs.h> #include <linux/slab.h> +#include <linux/vmstat.h> #include <asm/scs.h> static struct kmem_cache *scs_cache;
@@ -41,6 +43,17 @@ void __init scs_init(void) 0, NULL); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return virt_to_page (__scs_base (tsk)); +} + +static void scs_account (struct task_struct *tsk,int account) +{ +mod_zone_page_state (page_zone (__scs_page (tsk)),NR_KERNEL_SCS_KB , + account * (SCS_SIZE /1024 )); +} +int scs_prepare (struct task_struct *tsk,int node) {void *s;
@@ -50,6 +63,8 @@ int scs_prepare(struct task_struct *tsk, int node)return -ENOMEM ;task_set_scs (tsk, s); +scs_account (tsk,1 ); +return 0 ; }
@@ -63,5 +78,6 @@ void scs_release(struct task_struct *tsk)WARN_ON (scs_corrupted (tsk)); +scs_account (tsk,-1 );scs_free (s); }
mm/page_alloc.c +6 -0
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " managed:%lukB" " mlocked:%lukB" " kernel_stack:%lukB" +#ifdef CONFIG_SHADOW_CALL_STACK + " shadow_call_stack:%lukB" +#endif" pagetables:%lukB" " bounce:%lukB" " free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)K (zone_managed_pages (zone)),K (zone_page_state (zone,NR_MLOCK )),zone_page_state (zone,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK +zone_page_state (zone,NR_KERNEL_SCS_KB ), +#endifK (zone_page_state (zone,NR_PAGETABLE )),K (zone_page_state (zone,NR_BOUNCE )), K(free_pcp),
mm/vmstat.c +3 -0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {"nr_mlock" ,"nr_page_table_pages" ,"nr_kernel_stack" , +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +"nr_shadow_call_stack" , +#endif"nr_bounce" , #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages",
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Re: [PATCH v12 02/12] scs: add accounting
Quoted 4 lines · Sami Tolvanen
This change adds accounting for the memory allocated for shadow stacks.
Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>
Quoted 8 lines
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 16 ++++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 38 insertions(+)
Thanks!
Will
PATCH v12 3/12 scs: add support for stack usage debugging
In reply to Sami TolvanenImplements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.
--- kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+)
kernel/scs.c +38 -0
@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static unsigned long __scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =__scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static unsigned long highest; +unsigned long used =__scs_used (tsk); +unsigned long prev; +unsigned long curr = highest; + +while (used > curr) { + prev =cmpxchg (&highest, curr, used); + +if (prev == curr) { +pr_info ("%s (%d): highest shadow stack usage: " +"%lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); +break ; + } + + curr = prev; + } +} +#else +static inline void scs_check_usage (struct task_struct *tsk) {} +#endif +void scs_release (struct task_struct *tsk) {void *s;
@@ -77,6 +114,7 @@ void scs_release(struct task_struct *tsk)return ;WARN_ON (scs_corrupted (tsk)); +scs_check_usage (tsk);scs_account (tsk,-1 );scs_free (s);
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Re: [PATCH v12 03/12] scs: add support for stack usage debugging
Quoted 29 lines · Sami Tolvanen
Implements
CONFIG_DEBUG_STACK_USAGEfor shadow stacks. When enabled, also prints out the highest shadow stack usage per process.Signed-off-by Sami Tolvanen <[email protected]>Reviewed-by Kees Cook <[email protected]>--- kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+)kernel/scs.c +12
-0@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static unsigned long __scs_used (struct task_struct *tsk) +{ +unsigned long *p =__scs_base (tsk); +unsigned long *end =__scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +}
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 92
Quoted 10 lines
kernel/scs.c +10
-0+ +static void scs_check_usage (struct task_struct *tsk) +{ +static unsigned long highest; +unsigned long used =__scs_used (tsk); +unsigned long prev; +unsigned long curr = highest; + +while (used > curr) { + prev =cmpxchg (&highest, curr, used);
I think this can be cmpxchg_relaxed(), since we don't care about ordering here afaict.
With that:
Cheers,
Will
Re: [PATCH v12 03/12] scs: add support for stack usage debugging
Quoted 4 lines · Will Deacon
Quoted 4 lines
kernel/scs.c +4
-0+static void scs_check_usage (struct task_struct *tsk) +{ +static unsigned long highest; +unsigned long used =__scs_used (tsk);
Quoted 8 lines
kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 92
Quoted 5 lines
kernel/scs.c +5
-0+unsigned long prev; +unsigned long curr = highest; + +while (used > curr) { + prev =cmpxchg (&highest, curr, used);I think this can be
cmpxchg_relaxed(), since we don't care about ordering here afaict.
Sure, I'll change this in v13. Thanks.
Sami
PATCH v12 4/12 scs: disable when function graph tracing is enabled
In reply to Sami TolvanenThe graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
@@ -543,6 +543,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER help This option enables Clang's Shadow Call Stack, which uses a shadow stack to protect function return addresses from being
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 5/12 arm64: reserve x18 from general allocation with SCS
In reply to Sami TolvanenReserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.
--- arch/arm64/Makefile | 4 ++++ 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
@@ -81,6 +81,10 @@ endif KBUILD_CFLAGS += $(branch-prot-flags-y) +ifeq ($(CONFIG_SHADOW_CALL_STACK), y) +KBUILD_CFLAGS += -ffixed-x18 +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 6/12 arm64: preserve x18 when CPU is suspended
In reply to Sami TolvanenDon't lose the current task's shadow stack when the CPU is suspended.
--- arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/mm/proc.S | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@#ifndef __ASM_SUSPEND_H#define __ASM_SUSPEND_H -#define NR_CTX_REGS 12 +#define NR_CTX_REGS 13#define NR_CALLEE_SAVED_REGS 12/*
arch/arm64/mm/proc.S +14 -0
@@ -58,6 +58,8 @@ * cpu_do_suspend - save CPU registers context * * x0: virtual address of context pointer + * + * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. */ SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif stp x8, x9, [x0, #48] stp x10, x11, [x0, #64] stp x12, x13, [x0, #80] + /* + * Save x18 as it may be used as a platform register, e.g. by shadow + * call stack. + */ + str x18, [x0, #96] ret SYM_FUNC_END(cpu_do_suspend)
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + /* + * Restore x18, as it may be used as a platform register, and clear + * the buffer to minimize the risk of exposure when used for shadow + * call stack. + */ + ldr x18, [x0, #96] + str xzr, [x0, #96] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 7/12 arm64: efi: restore x18 if it was corrupted
In reply to Sami TolvanenIf we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.
--- arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper) ldp x29, x30, [sp], #32 b.ne 0f ret -0: b efi_handle_corrupted_x18 // tail call +0: + /* + * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a + * shadow stack pointer, which we need to restore before returning to + * potentially instrumented code. This is safe because the wrapper is + * called with preemption disabled and a separate shadow stack is used + * for interrupts. + */ + mov x18, x2 + b efi_handle_corrupted_x18 // tail call ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 8/12 arm64: vdso: disable Shadow Call Stack
In reply to Sami TolvanenShadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.
--- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING VDSO_LDFLAGS := -Bsymbolic -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) KBUILD_CFLAGS += $(DISABLE_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 9/12 arm64: disable SCS for hypervisor code
In reply to Sami TolvanenDisable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.
--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 10/12 arm64: implement Shadow Call Stack
In reply to Sami TolvanenThis change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
--- arch/arm64/Kconfig | 5 ++++ arch/arm64/include/asm/scs.h | 34 ++++++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 3 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 33 +++++++++++++++++++++++++-- arch/arm64/kernel/head.S | 8 +++++++ arch/arm64/kernel/process.c | 2 ++ arch/arm64/kernel/scs.c | 16 +++++++++++++ arch/arm64/kernel/smp.c | 4 ++++ 10 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
@@ -64,6 +64,7 @@ config ARM64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 +# Supported by clang >= 7.0 +config CC_HAVE_SHADOW_CALL_STACK + def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help---
arch/arm64/include/asm/scs.h new file +34 -0
@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SCS_H +#define _ASM_SCS_H + +#ifndef __ASSEMBLY__ + +#include <linux/scs.h> + +#ifdef CONFIG_SHADOW_CALL_STACK + +static __always_inline void scs_save(struct task_struct *tsk) +{ +void *s; + + asmvolatile ("mov %0, x18" :"=r" (s)); +task_set_scs (tsk, s); +} + +static inline void scs_overflow_check (struct task_struct *tsk) +{ +if (unlikely (scs_corrupted (tsk))) +panic ("corrupted shadow stack detected inside scheduler\n" ); +} + +#else /* CONFIG_SHADOW_CALL_STACK */ + +static inline void scs_save (struct task_struct *tsk) {} +static inline void scs_overflow_check (struct task_struct *tsk) {} + +#endif /* CONFIG_SHADOW_CALL_STACK */ + +#endif /* __ASSEMBLY __ */ + +#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
@@ -41,6 +41,9 @@ struct thread_info {#endif } preempt; }; +#ifdef CONFIG_SHADOW_CALL_STACK +void *shadow_call_stack; +#endif };#define thread_saved_pc (tsk) \
arch/arm64/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_SSBD) += ssbd.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-y += vdso/ probes/ obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
@@ -33,6 +33,9 @@ int main(void) DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));#ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif +#ifdef CONFIG_SHADOW_CALL_STACK + DEFINE(TSK_TI_SCS, offsetof(struct task_struct, thread_info.shadow_call_stack));#endif DEFINE (TSK_STACK , offsetof(struct task_struct , stack));#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
@@ -179,6 +179,11 @@ alternative_cb_end apply_ssbd 1, x22, x23 ptrauth_keys_install_kernel tsk, 1, x20, x22, x23 + +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [tsk, #TSK_TI_SCS] // Restore shadow call stack + str xzr, [tsk, #TSK_TI_SCS] // Limit visibility of saved SCS +#endif .else add x21, sp, #S_FRAME_SIZE get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif ct_user_enter .endif +#ifdef CONFIG_SHADOW_CALL_STACK + .if \el == 0 + str x18, [tsk, #TSK_TI_SCS] // Save shadow call stack + .endif +#endif + #ifdef CONFIG_ARM64_SW_TTBR0_PAN /* * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .macro irq_stack_entry mov x19, sp // preserve the original sp +#ifdef CONFIG_SHADOW_CALL_STACK + mov x24, x18 // preserve the original shadow stack +#endif /* * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 /* switch to the irq stack */ mov sp, x26 + +#ifdef CONFIG_SHADOW_CALL_STACK + /* also switch to the irq shadow stack */ + adr_this_cpu x18, irq_shadow_call_stack, x26 +#endif + 9998: .endm /* - * x19 should be preserved between irq_stack_entry and - * irq_stack_exit. + * The callee-saved regs (x19-x29) should be preserved between + * irq_stack_entry and irq_stack_exit, but note that kernel_entry + * uses x20-x23 to store data for later use. */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_SHADOW_CALL_STACK + mov x18, x24 +#endif .endm /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to) mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, 1, x8, x9, x10 +#ifdef CONFIG_SHADOW_CALL_STACK + str x18, [x0, #TSK_TI_SCS] + ldr x18, [x1, #TSK_TI_SCS] + str xzr, [x1, #TSK_TI_SCS] // limit visibility of saved SCS +#endif ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) stp xzr, x30, [sp, #-16]! mov x29, sp +#ifdef CONFIG_SHADOW_CALL_STACK + adr_l x18, init_shadow_call_stack // Set shadow call stack +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched) ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow msr sp_el0, x2 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr x18, [x2, #TSK_TI_SCS] // set shadow call stack + str xzr, [x2, #TSK_TI_SCS] // limit visibility of saved SCS +#endif mov x29, #0 mov x30, #0 b secondary_start_kernel
arch/arm64/kernel/process.c +2 -0
@@ -52,6 +52,7 @@#include <asm/mmu_context.h> #include <asm/processor.h> #include <asm/pointer_auth.h> +#include <asm/scs.h> #include <asm/stacktrace.h> #if defined(CONFIG_STACKPROTECTOR ) && !defined(CONFIG_STACKPROTECTOR_PER_TASK )
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,entry_task_switch (next);uao_thread_switch (next);ssbs_thread_switch (next); +scs_overflow_check (next);/* * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +16 -0
@@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/percpu.h> +#include <asm/scs.h> + +/* Allocate a static per-CPU shadow stack */ +#define DEFINE_SCS (name) \ + DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \ + __aligned(SCS_SIZE) + +DEFINE_SCS (irq_shadow_call_stack);
arch/arm64/kernel/smp.c +4 -0
@@ -46,6 +46,7 @@#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/processor.h> +#include <asm/scs.h> #include <asm/smp_plat.h> #include <asm/sections.h> #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)unsigned int cpu =smp_processor_id ();const struct cpu_operations *ops =get_cpu_ops (cpu); +/* Save the shadow stack pointer before exiting the idle task */ +scs_save (current); +idle_task_exit ();local_daif_mask ();
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 11/12 arm64: scs: add shadow stacks for SDEI
In reply to Sami TolvanenThis change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.
--- arch/arm64/kernel/entry.S | 14 +++++++++++++- arch/arm64/kernel/scs.c | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-)
arch/arm64/kernel/entry.S +13 -1
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler) mov x19, x1 +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] +#endif + #ifdef CONFIG_VMAP_STACK /* * entry.S may have been using sp as a scratch register, find whether * this is a normal or critical event and switch to the appropriate * stack for this CPU. */ - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] cbnz w4, 1f ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 b 2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler) mov sp, x5 #endif +#ifdef CONFIG_SHADOW_CALL_STACK + /* Use a separate shadow call stack for normal and critical events */ + cbnz w4, 3f + adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal, tmp=x6 + b 4f +3: adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical, tmp=x6 +4: +#endif + /* * We may have interrupted userspace, or a guest, or exit-from or * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +5 -0
@@ -14,3 +14,8 @@__aligned(SCS_SIZE) DEFINE_SCS (irq_shadow_call_stack); + +#ifdef CONFIG_ARM_SDE_INTERFACE +DEFINE_SCS (sdei_shadow_call_stack_normal); +DEFINE_SCS (sdei_shadow_call_stack_critical); +#endif
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
PATCH v12 12/12 efi/libstub: disable SCS
In reply to Sami TolvanenShadow stacks are not available in the EFI stub, filter out SCS flags.
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
v13 cover add support for Clang's Shadow Call Stack
In reply to Sami TolvanenThis patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Changes in v13:
- Changed thread_info::shadow_call_stack to a base address and an offset instead, and removed the now unneeded __scs_base() and scs_save().
- Removed alignment from the kmem_cache and static allocations.
- Removed the task_set_scs() helper function.
- Moved the assembly code for loading and storing the offset in thread_info to scs_load/save macros.
- Added offset checking to scs_corrupted().
- Switched to cmpxchg_relaxed() in scs_check_usage().
Changes in v12:
- Removed CONFIG_SHADOW_CALL_STACK_VMAP.
- Added CC_IS_CLANG as a dependency to CONFIG_SHADOW_CALL_STACK.
- Changed SCS_END_MAGIC to use POISON_POINTER_DELTA.
- Removed the unnecessary scs_set_magic() helper function.
- Moved scs_task_reset() and scs_corrupted() to scs.h, along with __scs_magic() and __scs_base().
- Removed a redundant warning from memory allocation.
- Removed an unnecessary task_set_scs() call from scs_release().
- Changed the accounting code to calculate KiB instead of bytes.
- Replaced the lock in scs_check_usage() with a cmpxchg() loop.
Changes in v11:
- Rebased, added maintainers for kernel/ changes.
Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.
Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.
Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.
Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.
Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler
Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information
Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical
Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit
Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions
Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS
Makefile | 6 ++ arch/Kconfig | 25 ++++++ arch/arm64/Kconfig | 5 ++ arch/arm64/Makefile | 4 + arch/arm64/include/asm/kvm_hyp.h | 2 +- arch/arm64/include/asm/scs.h | 46 ++++++++++ arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/include/asm/thread_info.h | 13 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 4 + arch/arm64/kernel/efi-rt-wrapper.S | 11 ++- arch/arm64/kernel/entry.S | 38 +++++++- arch/arm64/kernel/head.S | 6 ++ arch/arm64/kernel/process.c | 2 + arch/arm64/kernel/scs.c | 20 +++++ arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/mm/proc.S | 14 +++ drivers/base/node.c | 6 ++ drivers/firmware/efi/libstub/Makefile | 3 + fs/proc/meminfo.c | 4 + include/linux/compiler-clang.h | 6 ++ include/linux/compiler_types.h | 4 + include/linux/mmzone.h | 3 + include/linux/scs.h | 72 +++++++++++++++ init/init_task.c | 8 ++ kernel/Makefile | 1 + kernel/fork.c | 9 ++ kernel/sched/core.c | 2 + kernel/scs.c | 121 ++++++++++++++++++++++++++ mm/page_alloc.c | 6 ++ mm/vmstat.c | 3 + 31 files changed, 442 insertions(+), 7 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
base-commit: 6a8b55ed4056ea5559ebe4f6a4b247f627870d4c
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 1/12 add support for Clang's Shadow Call Stack (SCS)
This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:
https://clang.llvm.org/docs/ShadowCallStack.html
Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the stacks.
--- Makefile | 6 +++ arch/Kconfig | 24 ++++++++++++ include/linux/compiler-clang.h | 6 +++ include/linux/compiler_types.h | 4 ++ include/linux/scs.h | 72 ++++++++++++++++++++++++++++++++++ init/init_task.c | 8 ++++ kernel/Makefile | 1 + kernel/fork.c | 9 +++++ kernel/sched/core.c | 2 + kernel/scs.c | 68 ++++++++++++++++++++++++++++++++ 10 files changed, 200 insertions(+) create mode 100644 include/linux/scs.h create mode 100644 kernel/scs.c
Makefile +6 -0
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif +ifdef CONFIG_SHADOW_CALL_STACK +CC_FLAGS_SCS := -fsanitize=shadow-call-stack +KBUILD_CFLAGS += $(CC_FLAGS_SCS) +export CC_FLAGS_SCS +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
arch/Kconfig +24 -0
@@ -533,6 +533,30 @@ config STACKPROTECTOR_STRONG about 20% of all kernel functions, which increases the kernel code size by about 2%. +config ARCH_SUPPORTS_SHADOW_CALL_STACK + bool + help + An architecture should select this if it supports Clang's Shadow + Call Stack, has asm/scs.h, and implements runtime support for shadow + stack switching. + +config SHADOW_CALL_STACK + bool "Clang Shadow Call Stack" + depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + help + This option enables Clang's Shadow Call Stack, which uses a + shadow stack to protect function return addresses from being + overwritten by an attacker. More information can be found in + Clang's documentation: + + https://clang.llvm.org/docs/ShadowCallStack.html + + Note that security guarantees in the kernel differ from the + ones documented for user space. The kernel must store addresses + of shadow stacks in memory, which means an attacker capable of + reading and writing arbitrary memory may be able to locate them + and hijack control flow by modifying the stacks. + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help
include/linux/compiler-clang.h +6 -0
@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif
include/linux/compiler_types.h +4 -0
@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif +#ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x)#endif
include/linux/scs.h new file +72 -0
72 lines not on this page · raw message
init/init_task.c +8 -0
@@ -11,6 +11,7 @@#include <linux/mm.h> #include <linux/audit.h> #include <linux/numa.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <linux/uaccess.h>
@@ -50,6 +51,13 @@ static struct sighand_struct init_sighand = { .signalfd_wqh = __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh), }; +#ifdef CONFIG_SHADOW_CALL_STACK +unsigned long init_shadow_call_stack[SCS_SIZE /sizeof (long )] + __init_task_data = { + [(SCS_SIZE /sizeof (long )) -1 ] =SCS_END_MAGIC +}; +#endif +/* * Set up the first task table, touch at your own risk!. Base=0, * limit=0x1fffff (=2MB)
kernel/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/fork.c +9 -0
@@ -94,6 +94,7 @@#include <linux/thread_info.h> #include <linux/stackleak.h> #include <linux/kasan.h> +#include <linux/scs.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)void free_task (struct task_struct *tsk) { +scs_release (tsk); +#ifndef CONFIG_THREAD_INFO_IN_TASK /* * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void) NULL, free_vm_stack_cache);#endif +scs_init (); +lockdep_init_task (&init_task);uprobes_init (); }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)if (err) goto free_stack; + err =scs_prepare (tsk, node); +if (err) + goto free_stack; +#ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
@@ -11,6 +11,7 @@#include <linux/nospec.h> #include <linux/kcov.h> +#include <linux/scs.h> #include <asm/switch_to.h> #include <asm/tlb.h>
@@ -6040,6 +6041,7 @@ void init_idle(struct task_struct *idle, int cpu) idle->se.exec_start =sched_clock (); idle->flags |=PF_IDLE ; +scs_task_reset (idle);kasan_unpoison_task_stack (idle);#ifdef CONFIG_SMP
kernel/scs.c new file +68 -0
68 lines not on this page · raw message
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
Re: [PATCH v13 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 6 lines · Sami Tolvanen
include/linux/compiler-clang.h +0
-0
include/linux/compiler-clang.h line 50
Quoted 9 lines
include/linux/compiler-clang.h +6
-0@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") + +#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif
Can we remove the #else branch? compiler_types.h [*] has to care anyway about that case for other compilers anyway, no?
include/linux/compiler_types.h struct ftrace_likely_data { · line 198
Quoted 11 lines
include/linux/compiler_types.h +3
-0@@ -193,6 +193,10 @@ struct ftrace_likely_data {# define randomized_struct_fields_end#endif +#ifndef __noscs +# define __noscs +#endif
[*] Here
Cheers,
Miguel
Re: [PATCH v13 01/12] add support for Clang's Shadow Call Stack (SCS)
Quoted 11 lines · Miguel Ojeda
Quoted 10 lines · Sami Tolvanen
include/linux/compiler-clang.h +1
-0@@ -42,3 +42,9 @@* compilers, like ICC. */ #define barrier () __asm__ __volatile__("" : : : "memory") +
Quoted 8 lines
include/linux/compiler-clang.h line 50
Quoted 5 lines
include/linux/compiler-clang.h +5
-0+#if __has_feature (shadow_call_stack) +# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) +#else +# define __noscs +#endif Can we remove the
#elsebranch?compiler_types.h[*] has to care anyway about that case for other compilers anyway, no?
Yes, it's unnecessary. I'll remove this in the next version. Thanks!
Sami
PATCH v13 2/12 scs: add accounting
In reply to Sami TolvanenThis change adds accounting for the memory allocated for shadow stacks.
--- drivers/base/node.c | 6 ++++++ fs/proc/meminfo.c | 4 ++++ include/linux/mmzone.h | 3 +++ kernel/scs.c | 15 +++++++++++++++ mm/page_alloc.c | 6 ++++++ mm/vmstat.c | 3 +++ 6 files changed, 37 insertions(+)
drivers/base/node.c +6 -0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d AnonPages: %8lu kB\n" "Node %d Shmem: %8lu kB\n" "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" +#endif"Node %d PageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, nid,K (node_page_state (pgdat,NR_ANON_MAPPED )), nid,K (i.sharedram), nid,sum_zone_node_page_state (nid,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK + nid,sum_zone_node_page_state (nid,NR_KERNEL_SCS_KB ), +#endif nid,K (sum_zone_node_page_state (nid,NR_PAGETABLE )), nid,K (node_page_state (pgdat,NR_UNSTABLE_NFS )), nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)show_val_kb (m,"SUnreclaim: " , sunreclaim);seq_printf (m,"KernelStack: %8lu kB\n" ,global_zone_page_state (NR_KERNEL_STACK_KB )); +#ifdef CONFIG_SHADOW_CALL_STACK +seq_printf (m,"ShadowCallStack:%8lu kB\n" , +global_zone_page_state (NR_KERNEL_SCS_KB )); +#endif show_val_kb (m,"PageTables: " ,global_zone_page_state (NR_PAGETABLE ));
include/linux/mmzone.h +3 -0
@@ -156,6 +156,9 @@ enum zone_stat_item {NR_MLOCK ,/* mlock()ed pages found and moved off LRU */ NR_PAGETABLE ,/* used for pagetables */ NR_KERNEL_STACK_KB ,/* measured in KiB */ +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +NR_KERNEL_SCS_KB ,/* measured in KiB */ +#endif/* Second 128 byte cacheline */ NR_BOUNCE , #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +15 -0
@@ -6,8 +6,10 @@*/ #include <linux/kasan.h> +#include <linux/mm.h> #include <linux/scs.h> #include <linux/slab.h> +#include <linux/vmstat.h> #include <asm/scs.h> static struct kmem_cache *scs_cache;
@@ -40,6 +42,17 @@ void __init scs_init(void) scs_cache =kmem_cache_create ("scs_cache" ,SCS_SIZE ,0 ,0 ,NULL ); } +static struct page *__scs_page (struct task_struct *tsk) +{ +return virt_to_page (task_scs (tsk)); +} + +static void scs_account (struct task_struct *tsk,int account) +{ +mod_zone_page_state (page_zone (__scs_page (tsk)),NR_KERNEL_SCS_KB , + account * (SCS_SIZE /1024 )); +} +int scs_prepare (struct task_struct *tsk,int node) {void *s;
@@ -50,6 +63,7 @@ int scs_prepare(struct task_struct *tsk, int node)task_scs (tsk) = s;task_scs_offset (tsk) =0 ; +scs_account (tsk,1 );return 0 ; }
@@ -64,5 +78,6 @@ void scs_release(struct task_struct *tsk)WARN_ON (scs_corrupted (tsk)); +scs_account (tsk,-1 );scs_free (s); }
mm/page_alloc.c +6 -0
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " managed:%lukB" " mlocked:%lukB" " kernel_stack:%lukB" +#ifdef CONFIG_SHADOW_CALL_STACK + " shadow_call_stack:%lukB" +#endif" pagetables:%lukB" " bounce:%lukB" " free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)K (zone_managed_pages (zone)),K (zone_page_state (zone,NR_MLOCK )),zone_page_state (zone,NR_KERNEL_STACK_KB ), +#ifdef CONFIG_SHADOW_CALL_STACK +zone_page_state (zone,NR_KERNEL_SCS_KB ), +#endifK (zone_page_state (zone,NR_PAGETABLE )),K (zone_page_state (zone,NR_BOUNCE )), K(free_pcp),
mm/vmstat.c +3 -0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {"nr_mlock" ,"nr_page_table_pages" ,"nr_kernel_stack" , +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) +"nr_shadow_call_stack" , +#endif"nr_bounce" , #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages",
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 3/12 scs: add support for stack usage debugging
In reply to Sami TolvanenImplements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.
--- kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+)
kernel/scs.c +38 -0
@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)return 0 ; } +#ifdef CONFIG_DEBUG_STACK_USAGE +static unsigned long __scs_used (struct task_struct *tsk) +{ +unsigned long *p =task_scs (tsk); +unsigned long *end =__scs_magic (p); +unsigned long s = (unsigned long )p; + +while (p < end &&READ_ONCE_NOCHECK (*p)) + p++; + +return (unsigned long )p - s; +} + +static void scs_check_usage (struct task_struct *tsk) +{ +static unsigned long highest; +unsigned long used =__scs_used (tsk); +unsigned long prev; +unsigned long curr = highest; + +while (used > curr) { + prev =cmpxchg_relaxed (&highest, curr, used); + +if (prev == curr) { +pr_info ("%s (%d): highest shadow stack usage: " +"%lu bytes\n" , + tsk->comm,task_pid_nr (tsk), used); +break ; + } + + curr = prev; + } +} +#else +static inline void scs_check_usage (struct task_struct *tsk) {} +#endif +void scs_release (struct task_struct *tsk) {void *s;
@@ -77,6 +114,7 @@ void scs_release(struct task_struct *tsk)return ;WARN_ON (scs_corrupted (tsk)); +scs_check_usage (tsk);scs_account (tsk,-1 );scs_free (s);
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 4/12 scs: disable when function graph tracing is enabled
In reply to Sami TolvanenThe graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.
With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.
Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.
SCS and patchable-function-entry are both supported from LLVM 10.x.
--- arch/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
@@ -543,6 +543,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK config SHADOW_CALL_STACK bool "Clang Shadow Call Stack" depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK + depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER help This option enables Clang's Shadow Call Stack, which uses a shadow stack to protect function return addresses from being
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 5/12 arm64: reserve x18 from general allocation with SCS
In reply to Sami TolvanenReserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.
--- arch/arm64/Makefile | 4 ++++ 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
@@ -81,6 +81,10 @@ endif KBUILD_CFLAGS += $(branch-prot-flags-y) +ifeq ($(CONFIG_SHADOW_CALL_STACK), y) +KBUILD_CFLAGS += -ffixed-x18 +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 6/12 arm64: preserve x18 when CPU is suspended
In reply to Sami TolvanenDon't lose the current task's shadow stack when the CPU is suspended.
--- arch/arm64/include/asm/suspend.h | 2 +- arch/arm64/mm/proc.S | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@#ifndef __ASM_SUSPEND_H#define __ASM_SUSPEND_H -#define NR_CTX_REGS 12 +#define NR_CTX_REGS 13#define NR_CALLEE_SAVED_REGS 12/*
arch/arm64/mm/proc.S +14 -0
@@ -58,6 +58,8 @@ * cpu_do_suspend - save CPU registers context * * x0: virtual address of context pointer + * + * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. */ SYM_FUNC_START(cpu_do_suspend) mrs x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif stp x8, x9, [x0, #48] stp x10, x11, [x0, #64] stp x12, x13, [x0, #80] + /* + * Save x18 as it may be used as a platform register, e.g. by shadow + * call stack. + */ + str x18, [x0, #96] ret SYM_FUNC_END(cpu_do_suspend)
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume) ldp x9, x10, [x0, #48] ldp x11, x12, [x0, #64] ldp x13, x14, [x0, #80] + /* + * Restore x18, as it may be used as a platform register, and clear + * the buffer to minimize the risk of exposure when used for shadow + * call stack. + */ + ldr x18, [x0, #96] + str xzr, [x0, #96] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 7/12 arm64: efi: restore x18 if it was corrupted
In reply to Sami TolvanenIf we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.
--- arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper) ldp x29, x30, [sp], #32 b.ne 0f ret -0: b efi_handle_corrupted_x18 // tail call +0: + /* + * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a + * shadow stack pointer, which we need to restore before returning to + * potentially instrumented code. This is safe because the wrapper is + * called with preemption disabled and a separate shadow stack is used + * for interrupts. + */ + mov x18, x2 + b efi_handle_corrupted_x18 // tail call ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 8/12 arm64: vdso: disable Shadow Call Stack
In reply to Sami TolvanenShadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.
--- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING VDSO_LDFLAGS := -Bsymbolic -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) KBUILD_CFLAGS += $(DISABLE_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 9/12 arm64: disable SCS for hypervisor code
In reply to Sami TolvanenDisable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.
--- arch/arm64/include/asm/kvm_hyp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
@@ -13,7 +13,7 @@#include <asm/kvm_mmu.h> #include <asm/sysreg.h> -#define __hyp_text __section(.hyp.text) notrace +#define __hyp_text __section(.hyp.text) notrace __noscs#define read_sysreg_elx (r,nvh,vh) \ ({ \
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 10/12 arm64: implement Shadow Call Stack
In reply to Sami TolvanenThis change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.
--- arch/arm64/Kconfig | 5 +++ arch/arm64/include/asm/scs.h | 46 ++++++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 13 ++++++++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/asm-offsets.c | 4 +++ arch/arm64/kernel/entry.S | 24 +++++++++++++-- arch/arm64/kernel/head.S | 6 ++++ arch/arm64/kernel/process.c | 2 ++ arch/arm64/kernel/scs.c | 15 +++++++++ 9 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/scs.h create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
@@ -64,6 +64,7 @@ config ARM64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 +# Supported by clang >= 7.0 +config CC_HAVE_SHADOW_CALL_STACK + def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18) + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help---
arch/arm64/include/asm/scs.h new file +46 -0
46 lines not on this page · raw message
arch/arm64/include/asm/thread_info.h +13 -0
@@ -41,6 +41,10 @@ struct thread_info {#endif } preempt; }; +#ifdef CONFIG_SHADOW_CALL_STACK +void *scs_base; +unsigned long scs_offset; +#endif };#define thread_saved_pc (tsk) \
@@ -100,11 +104,20 @@ void arch_release_task_struct(struct task_struct *tsk); _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ _TIF_SYSCALL_EMU) +#ifdef CONFIG_SHADOW_CALL_STACK +#define INIT_SCS \ + .scs_base = init_shadow_call_stack, \ + .scs_offset = 0, +#else +#define INIT_SCS +#endif +#define INIT_THREAD_INFO (tsk) \ { \ .flags = _TIF_FOREIGN_FPSTATE, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ + INIT_SCS \ }#endif /* __ASM_THREAD_INFO_H */
arch/arm64/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_SSBD) += ssbd.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-y += vdso/ probes/ obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c +4 -0
@@ -33,6 +33,10 @@ int main(void) DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));#ifdef CONFIG_ARM64_SW_TTBR0_PAN DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0)); +#endif +#ifdef CONFIG_SHADOW_CALL_STACK + DEFINE(TSK_TI_SCS_BASE, offsetof(struct task_struct, thread_info.scs_base)); + DEFINE(TSK_TI_SCS_OFFSET, offsetof(struct task_struct, thread_info.scs_offset));#endif DEFINE (TSK_STACK , offsetof(struct task_struct , stack));#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +22 -2
@@ -23,6 +23,7 @@ #include <asm/mmu.h> #include <asm/processor.h> #include <asm/ptrace.h> +#include <asm/scs.h> #include <asm/thread_info.h> #include <asm/asm-uaccess.h> #include <asm/unistd.h>
@@ -179,6 +180,8 @@ alternative_cb_end apply_ssbd 1, x22, x23 ptrauth_keys_install_kernel tsk, 1, x20, x22, x23 + + scs_load tsk, x20 .else add x21, sp, #S_FRAME_SIZE get_current_task tsk
@@ -343,6 +346,8 @@ alternative_else_nop_endif msr cntkctl_el1, x1 4: #endif + scs_save tsk, x0 + /* No kernel C function calls after this as user keys are set. */ ptrauth_keys_install_user tsk, x0, x1, x2
@@ -388,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 .macro irq_stack_entry mov x19, sp // preserve the original sp +#ifdef CONFIG_SHADOW_CALL_STACK + mov x24, x18 // preserve the original shadow stack +#endif /* * Compare sp with the base of the task stack.
@@ -405,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 /* switch to the irq stack */ mov sp, x26 + +#ifdef CONFIG_SHADOW_CALL_STACK + /* also switch to the irq shadow stack */ + adr_this_cpu x18, irq_shadow_call_stack, x26 +#endif + 9998: .endm /* - * x19 should be preserved between irq_stack_entry and - * irq_stack_exit. + * The callee-saved regs (x19-x29) should be preserved between + * irq_stack_entry and irq_stack_exit, but note that kernel_entry + * uses x20-x23 to store data for later use. */ .macro irq_stack_exit mov sp, x19 +#ifdef CONFIG_SHADOW_CALL_STACK + mov x18, x24 +#endif .endm /* GPRs used by entry code */
@@ -901,6 +919,8 @@ SYM_FUNC_START(cpu_switch_to) mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, 1, x8, x9, x10 + scs_save x0, x8 + scs_load x1, x8 ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +6 -0
@@ -27,6 +27,7 @@ #include <asm/pgtable-hwdef.h> #include <asm/pgtable.h> #include <asm/page.h> +#include <asm/scs.h> #include <asm/smp.h> #include <asm/sysreg.h> #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) stp xzr, x30, [sp, #-16]! mov x29, sp +#ifdef CONFIG_SHADOW_CALL_STACK + adr_l x18, init_shadow_call_stack // Set shadow call stack +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between
@@ -737,6 +742,7 @@ SYM_FUNC_START_LOCAL(__secondary_switched) ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow msr sp_el0, x2 + scs_load x2, x3 mov x29, #0 mov x30, #0 b secondary_start_kernel
arch/arm64/kernel/process.c +2 -0
@@ -52,6 +52,7 @@#include <asm/mmu_context.h> #include <asm/processor.h> #include <asm/pointer_auth.h> +#include <asm/scs.h> #include <asm/stacktrace.h> #if defined(CONFIG_STACKPROTECTOR ) && !defined(CONFIG_STACKPROTECTOR_PER_TASK )
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,entry_task_switch (next);uao_thread_switch (next);ssbs_thread_switch (next); +scs_overflow_check (next);/* * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +15 -0
@@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Shadow Call Stack support. +* +* Copyright (C) 2019 Google LLC +*/ + +#include <linux/percpu.h> +#include <asm/scs.h> + +/* Allocate a static per-CPU shadow stack */ +#define DEFINE_SCS (name) \ + DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \ + +DEFINE_SCS (irq_shadow_call_stack);
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 11/12 arm64: scs: add shadow stacks for SDEI
In reply to Sami TolvanenThis change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.
--- arch/arm64/kernel/entry.S | 14 +++++++++++++- arch/arm64/kernel/scs.c | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-)
arch/arm64/kernel/entry.S +13 -1
@@ -1049,13 +1049,16 @@ SYM_CODE_START(__sdei_asm_handler) mov x19, x1 +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] +#endif + #ifdef CONFIG_VMAP_STACK /* * entry.S may have been using sp as a scratch register, find whether * this is a normal or critical event and switch to the appropriate * stack for this CPU. */ - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] cbnz w4, 1f ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 b 2f
@@ -1065,6 +1068,15 @@ SYM_CODE_START(__sdei_asm_handler) mov sp, x5 #endif +#ifdef CONFIG_SHADOW_CALL_STACK + /* Use a separate shadow call stack for normal and critical events */ + cbnz w4, 3f + adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal, tmp=x6 + b 4f +3: adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical, tmp=x6 +4: +#endif + /* * We may have interrupted userspace, or a guest, or exit-from or * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +5 -0
@@ -13,3 +13,8 @@ DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \DEFINE_SCS (irq_shadow_call_stack); + +#ifdef CONFIG_ARM_SDE_INTERFACE +DEFINE_SCS (sdei_shadow_call_stack_normal); +DEFINE_SCS (sdei_shadow_call_stack_critical); +#endif
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
PATCH v13 12/12 efi/libstub: disable SCS
In reply to Sami TolvanenShadow stacks are not available in the EFI stub, filter out SCS flags.
--- drivers/firmware/efi/libstub/Makefile | 3 +++ 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-stack-protector) \ -D__DISABLE_EXPORTS +# remove SCS flags from all objects in this directory +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) + GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
Ard Biesheuvel
maintainerQuoted 34 lines · Sami Tolvanen
This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Quoted 11 lines
Changes in v13:
- Changedthread_info::shadow_call_stackto a base address and an offset instead, and removed the now unneeded__scs_base()andscs_save().
- Removed alignment from thekmem_cacheand static allocations.
- Removed thetask_set_scs()helper function.
- Moved the assembly code for loading and storing the offset inthread_infoto scs_load/save macros.
- Added offset checking toscs_corrupted().
- Switched tocmpxchg_relaxed()inscs_check_usage().
OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.
The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.
Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.
Ard Biesheuvel
maintainerQuoted 58 lines · Ard Biesheuvel
Quoted 34 lines · Sami Tolvanen
This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
https://clang.llvm.org/docs/ShadowCallStack.html
SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.
SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.
With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.
SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.
Quoted 11 lines
Changes in v13:
- Changedthread_info::shadow_call_stackto a base address and an offset instead, and removed the now unneeded__scs_base()andscs_save().
- Removed alignment from thekmem_cacheand static allocations.
- Removed thetask_set_scs()helper function.
- Moved the assembly code for loading and storing the offset inthread_infoto scs_load/save macros.
- Added offset checking toscs_corrupted().
- Switched tocmpxchg_relaxed()inscs_check_usage().OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.
The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.
Quoted 7 lines
Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.
Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(
Quoted 18 lines · Ard Biesheuvel
Quoted 10 lines
OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.
The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.
Quoted 7 lines
Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.
Quoted 2 lines
Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(
Yes, and the stack potentially overflowing into the SCS sort of defeats the purpose. I'm fine with increasing the SCS size to something safer, but using a vmapped shadow stack seems like the correct solution to this problem, at least on devices where allocating a full page isn't an issue.
Sami
From: Sami Tolvanen
Quoted 1 line
Sent: 27 April 2020 23:10
...
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.
Quoted 7 lines
Quoted 2 lines
Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(
Yes, and the stack potentially overflowing into the SCS sort of defeats the purpose. I'm fine with increasing the SCS size to something safer, but using a vmapped shadow stack seems like the correct solution to this problem, at least on devices where allocating a full page isn't an issue.
Wouldn't you do it the other way around - so shadow stack overflow corrupts the bottom of the normal stack?
That can be detected 'after the fact' in a few places (eg process switch and return to user)
Actually you might want to do syscall entry at the base of stack area, then (effectively) allocate an on-stack buffer for the shadow stack.
I'd have though that kernel code could be the shadow stack address by just reading r18?
Userspace isn't supposed to be able to get the main kernel stack address either.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Will Deacon
maintainerHi Sami,
Quoted 6 lines · Sami Tolvanen
This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:
I'm planning to queue this with the (mostly cosmetic) diff below folded in. I also have some extra patches on top which I'll send out shortly for review.
However, I really think we need to get to the bottom of the size issue since I'm highly sceptical about not being able to afford a full page for the shadow stack allocation. We can change this later so it needn't hold up the patchset, but given that Android is the only user, I'd like to make sure that if we change to use a full page upstream then that is also acceptable in AOSP.
Thanks,
Will
--->8
include/linux/compiler-clang.h +0 -2
@@ -45,6 +45,4 @@#if __has_feature (shadow_call_stack)# define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) -#else -# define __noscs#endif
include/linux/scs.h +5 -3
@@ -11,7 +11,7 @@#include <linux/gfp.h> #include <linux/poison.h> #include <linux/sched.h> -#include <asm/page.h> +#include <linux/sizes.h> #ifdef CONFIG_SHADOW_CALL_STACK
@@ -20,7 +20,7 @@* architecture) provided ~40% safety margin on stack usage while keeping * memory allocation overhead reasonable. */ -#define SCS_SIZE 1024UL +#define SCS_SIZE SZ_1K#define GFP_SCS (GFP_KERNEL | __GFP_ZERO)/* An illegal pointer value to mark the end of the shadow stack. */
@@ -29,7 +29,9 @@#define task_scs (tsk) (task_thread_info(tsk)->scs_base)#define task_scs_offset (tsk) (task_thread_info(tsk)->scs_offset) -extern void scs_init (void ); +void scs_init (void ); +int scs_prepare (struct task_struct *tsk,int node); +void scs_release (struct task_struct *tsk);static inline void scs_task_reset (struct task_struct *tsk) {
@@ -40,8 +42,6 @@ static inline void scs_task_reset(struct task_struct *tsk)
task_scs_offset(tsk) = 0;
}
-extern int scs_prepare(struct task_struct *tsk, int node); -
static inline unsigned long *__scs_magic(void *s)
{
return (unsigned long *)(s + SCS_SIZE) - 1;
@@ -55,12 +55,8 @@ static inline bool scs_corrupted(struct task_struct *tsk)
READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC);}
-extern void scs_release(struct task_struct *tsk); -
#else /* CONFIG_SHADOW_CALL_STACK */
-#define task_scs(tsk) NULL
-
static inline void scs_init(void) {}
static inline void scs_task_reset(struct task_struct *tsk) {} static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
kernel/scs.c +1 -2
@@ -55,45 +55,37 @@ static void scs_account(struct task_struct *tsk, int account)int scs_prepare (struct task_struct *tsk,int node) { -void *s; +void *s =scs_alloc (node); - s =scs_alloc (node);
if (!s) return -ENOMEM; task_scs(tsk) = s; task_scs_offset(tsk) = 0; scs_account(tsk, 1); - return 0;
}
-#ifdef CONFIG_DEBUG_STACK_USAGE
-static unsigned long __scs_used(struct task_struct *tsk) +static void scs_check_usage(struct task_struct *tsk)
{
- unsigned long *p = task_scs(tsk);
- unsigned long *end = __scs_magic(p);
- unsigned long s = (unsigned long)p;
+ static unsigned long highest;
- while (p < end && READ_ONCE_NOCHECK(*p))
- p++;
+ unsigned long *p, prev, curr = highest, used = 0;
- return (unsigned long)p - s;
-}
+ if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
+ return;
-static void scs_check_usage(struct task_struct *tsk) -{
- static unsigned long highest;
- unsigned long used = __scs_used(tsk);
- unsigned long prev;
- unsigned long curr = highest;
+ for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
+ if (!READ_ONCE_NOCHECK(*p))
+ break;
+ used++;
+ }
while (used > curr) {
prev = cmpxchg_relaxed(&highest, curr, used);
if (prev == curr) {
- pr_info("%s (%d): highest shadow stack usage: "
- "%lu bytes\n",
+ pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
tsk->comm, task_pid_nr(tsk), used);
break;
}
@@ -101,21 +93,16 @@ static void scs_check_usage(struct task_struct *tsk)
curr = prev;
} }
-#else
-static inline void scs_check_usage(struct task_struct *tsk) {} -#endif
void scs_release(struct task_struct *tsk)
{
- void *s;
+ void *s = task_scs(tsk);
- s = task_scs(tsk); if (!s) return;
- WARN_ON(scs_corrupted(tsk));
+ WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n");
scs_check_usage(tsk); - scs_account(tsk, -1); scs_free(s);
}