Skip to contentSkip to conversation

add support for Clang's Shadow Call Stack

PATCH Superseded Sami Tolvanen 18 participants 431 messages Oct 18, 2019 – May 15, 2020 Page 3 of 3

Subsystems (16)
lore
Share & tools
lore
#325 lore raw

PATCH v9 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 98a31bafc8a2..874a8b428438 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 462f6873905a..0a6f395abc68 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 3c4eb750a199..1381b9d84e4c 100644
@@ -5340,6 +5340,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5362,6 +5365,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 78d53378db99..d0650391c8c1 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#326 lore raw

PATCH v9 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#327 lore raw

PATCH v9 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index a67fa78c92e7..d53ade0950a5 100644
@@ -535,6 +535,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#328 lore raw

PATCH v9 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index dca1a97751ab..ab26b448faa9 100644
@@ -65,6 +65,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#329 lore raw

PATCH v9 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index aafed6902411..7d37e3c70ff5 100644
@@ -56,6 +56,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -80,6 +82,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -96,6 +103,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#330 lore raw

PATCH v9 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#331 lore raw

PATCH v9 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#332 lore raw

PATCH v9 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index a3a6a2ba9a63..0f0603f55ea0 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#333 lore raw

PATCH v9 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 32 ++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 0b30e884e088..eae76686be77 100644
@@ -65,6 +65,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1022,6 +1023,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index fc6488660f64..08fafc4da2cf 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index a5bdce8af65b..d485dc5cd196 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +30 -2
index 9461d812ae27..4b18c3bbdea5 100644
@@ -177,6 +177,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -400,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -895,6 +918,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..ca561de903d4 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 00626057a384..9151616c354c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -514,6 +515,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index d4ed9a19d8fe..f2cb344f998c 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -358,6 +359,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Reviewed-by on v9 10/12
#334 lore raw

Re: [PATCH v9 10/12] arm64: implement Shadow Call Stack

Hi Sami,

Quoted 9 lines · Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

arch/arm64/include/asm/scs.h new file +0 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f

arch/arm64/include/asm/scs.h line 5

Quoted 5 lines
arch/arm64/include/asm/scs.h new file +5 -0
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__

As the whole file is guarded by this, why do you need to include it in assembly files at all?

Quoted 9 lines
arch/arm64/include/asm/scs.h new file +9 -0
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{

arch/arm64/include/asm/scs.h line 24

Quoted 10 lines
arch/arm64/include/asm/scs.h new file +10 -0
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");

Could this ever catch anything with CONFIG_SHADOW_CALL_STACK_VMAP? Wouldn't we have hit the vmalloc guard page at the point of overflow?

arch/arm64/kernel/entry.S

Quoted 20 lines
arch/arm64/include/asm/scs.h new file +13 -0
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/kernel/entry.S +0 -0
index 9461d812ae27..4b18c3bbdea5 100644

If I corrupt x18 so that we take an exception (mov x18, xzr), we take that exception whenever we run C code. The CPU 'vanishes' and I get a very upset scheduler shortly after.

Stack misalignment has the same problem, but the overflow test (eventually) catches that, then calls panic() using the overflow stack. (See the kernel_ventry macro and __bad_stack in entry.S)

It would be nice to have a per-cpu stack that we switch to when on the overflow stack. (this would catch the scs overflow hitting the guard page too, as we should eat through the regular stack until we overflowed it!)

arch/arm64/kernel/smp.c void cpu_die(void) · line 362

Quoted 9 lines
arch/arm64/kernel/smp.c +1 -0
index d4ed9a19d8fe..f2cb344f998c 100644
@@ -358,6 +359,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */

I can't work out why this needs to be before before idle_task_exit()...
It needs to run before init_idle(), which calls scs_task_reset(), but all that is on the cpu_up() path. (if it is to pair those up, any reason core code can't do both?)

arch/arm64/kernel/smp.c void cpu_die(void) · line 367

Quoted 6 lines
arch/arm64/kernel/smp.c +2 -0
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();

Thanks!

James

#335 lore raw

Re: [PATCH v9 10/12] arm64: implement Shadow Call Stack

Quoted 3 lines · James Morse
Quoted 1 line

+#ifndef __ASSEMBLY__

As the whole file is guarded by this, why do you need to include it in assembly files at all?

True, the include in head.S is not needed. I'll remove it in the next version.

Quoted 7 lines

arch/arm64/include/asm/scs.h line 24

Quoted 4 lines
arch/arm64/include/asm/scs.h new file +4 -0
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+     if (unlikely(scs_corrupted(tsk)))
+             panic("corrupted shadow stack detected inside scheduler\n");

Could this ever catch anything with CONFIG_SHADOW_CALL_STACK_VMAP? Wouldn't we have hit the vmalloc guard page at the point of overflow?

With CONFIG_SHADOW_CALL_STACK_VMAP, even though we allocate a full page, SCS_SIZE is still 1k, so we should catch overflows here well before we hit the guard page.

Quoted 1 line

It would be nice to have a per-cpu stack that we switch to when on the overflow stack.

It shouldn't be a problem to add an overflow shadow stack if you think one is needed.

Quoted 3 lines

I can't work out why this needs to be before before idle_task_exit()...
It needs to run before init_idle(), which calls scs_task_reset(), but all that is on the cpu_up() path. (if it is to pair those up, any reason core code can't do both?)

At this point, the idle task's shadow stack pointer is only stored in x18, so we need to save it again to thread_info before the CPU shuts down, or we'll lose the pointer.

Sami

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#336 lore raw

PATCH v9 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index 4b18c3bbdea5..2e2ce1b9ebf5 100644
@@ -1050,13 +1050,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1066,6 +1069,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#337 lore raw

PATCH v9 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 98a81576213d..ee5c37c401c9 100644
@@ -30,6 +30,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#338 lore raw

v10 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.

Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 +
 arch/Kconfig                          |  35 ++++
 arch/arm64/Kconfig                    |   5 +
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  39 ++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |   3 +
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 +-
 arch/arm64/kernel/entry.S             |  47 ++++-
 arch/arm64/kernel/head.S              |   8 +
 arch/arm64/kernel/irq.c               |   2 +
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               | 114 ++++++++++++
 arch/arm64/kernel/sdei.c              |   7 +
 arch/arm64/kernel/smp.c               |   4 +
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 ++
 drivers/base/node.c                   |   6 +
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 +
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  57 ++++++
 init/init_task.c                      |   8 +
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 +
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 246 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 +
 mm/vmstat.c                           |   3 +
 34 files changed, 662 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: a10c9c710f9ecea87b9f4bbb837467893b4bef01

Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#339 lore raw

PATCH v10 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index c91342953d9e..cb2ed7443d57 100644
@@ -851,6 +851,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 786a85d4ad40..691a552c2cc3 100644
@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index bd403ed3e418..aaa71366d162 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index d2a967bf85d5..3f54070a7a53 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -455,6 +456,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -838,6 +841,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -897,6 +902,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index a2694ba82874..9bb593f7974f 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6050,6 +6051,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#340 lore raw

PATCH v10 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 10d7e818e118..502ab5447c8d 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index e84d448988b6..a6c60e6efa68 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index e5f76da8cd4e..79f07ccac63e 100644
@@ -5338,6 +5338,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5360,6 +5363,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index c9c0d71f917f..287a95987b7b 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#341 lore raw

PATCH v10 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#342 lore raw

PATCH v10 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 691a552c2cc3..c53cb9025ad2 100644
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#343 lore raw

PATCH v10 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index f15f92ba53e6..34277c60cdf9 100644
@@ -76,6 +76,10 @@ branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a
 KBUILD_CFLAGS += $(branch-prot-flags-y)
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#344 lore raw

PATCH v10 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index 197a9ba2d5ea..ed15be0f8103 100644
@@ -58,6 +58,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#345 lore raw

PATCH v10 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#346 lore raw

PATCH v10 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#347 lore raw

PATCH v10 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index fe57f60f06a8..875b106c5d98 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Reviewed-by on v10 9/12
#348 lore raw

Re: [PATCH v10 09/12] arm64: disable SCS for hypervisor code

Quoted 5 lines · Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

-Kees

Quoted 21 lines
---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index fe57f60f06a8..875b106c5d98 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Signature · 1 line
Kees Cook
#349 lore raw

PATCH v10 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 33 +++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  8 ++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 6e41c4b62607..b47c254ce1dd 100644
@@ -64,6 +64,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index 512174a8e789..1fb651f73da3 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index 4e5b8ee31442..151f28521f1e 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index 9981a0a5a87f..777a662888ec 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
index ddcde093c433..c33264ce7258 100644
@@ -179,6 +179,11 @@ alternative_cb_end
 	apply_ssbd 1, x22, x23
 
 	ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to)
 	mov	sp, x9
 	msr	sp_el0, x1
 	ptrauth_keys_install_kernel x1, 1, x8, x9, x10
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 SYM_FUNC_END(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
index 57a91032b4c2..1514445bbccb 100644
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 56be4cbf771f..a35d3318492c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	entry_task_switch(next);
 	uao_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index 061f60fe452f..1d112e34a636 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)
 	unsigned int cpu = smp_processor_id();
 	const struct cpu_operations *ops = get_cpu_ops(cpu);
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#350 lore raw

PATCH v10 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index c33264ce7258..768cd7abd32c 100644
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
#351 lore raw

PATCH v10 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 094eabdecfe6..fa0bb64f93d6 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Reviewed-by on v10 12/12
#352 lore raw

Re: [PATCH v10 12/12] efi/libstub: disable SCS

Quoted 4 lines · Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

drivers/firmware/efi/libstub/Makefile KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ · line 35

Quoted 13 lines
---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +1 -0
index 094eabdecfe6..fa0bb64f93d6 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory

nit: double space

-Kees

Quoted 8 lines
drivers/firmware/efi/libstub/Makefile +2 -0
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.0.292.g33ef6b2f38-goog
Signature · 6 lines
Kees Cook

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
maintainer Acked-by on v10 12/12
#353 lore raw

Re: [PATCH v10 12/12] efi/libstub: disable SCS

In reply to Sami Tolvanen
Quoted 5 lines · Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

Quoted 21 lines
---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -1
index 094eabdecfe6..fa0bb64f93d6 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS                 := $(cflags-y) -DDISABLE_BRANCH_PROFILING \
                                   $(call cc-option,-fno-stack-protector) \
                                   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE                   := n
 KASAN_SANITIZE                 := n
 UBSAN_SANITIZE                 := n
--

2.26.0.292.g33ef6b2f38-goog

#354 lore raw

v11 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v11:
- Rebased, added maintainers for kernel/ changes.

Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.

Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 +
 arch/Kconfig                          |  35 ++++
 arch/arm64/Kconfig                    |   5 +
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  39 ++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |   3 +
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 +-
 arch/arm64/kernel/entry.S             |  47 ++++-
 arch/arm64/kernel/head.S              |   8 +
 arch/arm64/kernel/irq.c               |   2 +
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               | 114 ++++++++++++
 arch/arm64/kernel/sdei.c              |   7 +
 arch/arm64/kernel/smp.c               |   4 +
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 ++
 drivers/base/node.c                   |   6 +
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 +
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  57 ++++++
 init/init_task.c                      |   8 +
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 +
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 246 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 +
 mm/vmstat.c                           |   3 +
 34 files changed, 662 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 00086336a8d96a04aa960f912287692a258f6cf5

Signature · 7 lines
2.26.1.301.g55bc3eb7cb9-goog


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
#355 lore raw

PATCH v11 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 70def4907036..baea6024b409 100644
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 786a85d4ad40..691a552c2cc3 100644
@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index bd403ed3e418..aaa71366d162 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 4385f3d639f2..c4c984d29573 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 3a61a3b8eaa9..c99620c1ec20 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer
#356 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Hi Sami,

Comments inline.

Quoted 36 lines · Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +0 -0
index 70def4907036..baea6024b409 100644

Makefile ifdef CONFIG_LIVEPATCH · line 873

Quoted 8 lines
Makefile +5 -0
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif

CFLAGS_SCS would seem more natural to me, although I see ftrace does it this way.

Quoted 25 lines
Makefile +0 -0
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +14 -0
index 786a85d4ad40..691a552c2cc3 100644
@@ -533,6 +533,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in

arch/Kconfig config STACKPROTECTOR_STRONG · line 559

Quoted 10 lines
arch/Kconfig +10 -0
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.

Shouldn't some of this depend on CC_IS_CLANG?

arch/Kconfig config STACKPROTECTOR_STRONG · line 567

Quoted 7 lines
arch/Kconfig +7 -0
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.

Given that this feature applies only to arm64 kernels built with clang, it feels weird to further segment that userbase with another config option. Does Android enable SHADOW_CALL_STACK_VMAP? If not, maybe we should ditch it for now and add it when we have a user.

Quoted 40 lines
include/linux/compiler_types.h +4 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +19 -0
new file mode 100644
index 000000000000..c5572fd770b0
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#ifndef _LINUX_SCS_H
+#define _LINUX_SCS_H
+
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <asm/page.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+/*
+ * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
+ * architecture) provided ~40% safety margin on stack usage while keeping

include/linux/scs.h line 29

Quoted 10 lines
include/linux/scs.h new file +10 -0
+ * memory allocation overhead reasonable.
+ */
+#define SCS_SIZE	1024UL
+#define GFP_SCS		(GFP_KERNEL | __GFP_ZERO)
+
+/*
+ * A random number outside the kernel's virtual address space to mark the
+ * end of the shadow stack.
+ */
+#define SCS_END_MAGIC	0xaf0194819b1635f6UL

This seems like it might be arm64-specific. Why not choose something based off CONFIG_ILLEGAL_POINTER_VALUE (see linux/poison.h)?

Quoted 33 lines
kernel/sched/core.c +2 -0
index 3a61a3b8eaa9..c99620c1ec20 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +7 -0
new file mode 100644
index 000000000000..28abed21950c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+

kernel/scs.c line 17

Quoted 10 lines
kernel/scs.c new file +10 -0
+#include <linux/cpuhotplug.h>
+#include <linux/kasan.h>
+#include <linux/mm.h>
+#include <linux/mmzone.h>
+#include <linux/scs.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <asm/scs.h>
+
+static inline void *__scs_base(struct task_struct *tsk)

Please avoid using 'inline' in C files unless there's a good reason not to let the compiler figure it out.

kernel/scs.c line 20

Quoted 3 lines
kernel/scs.c new file +3 -0
+{
+	/*
+	 * To minimize risk the of exposure, architectures may clear a

Should be "the risk of exposure".

kernel/scs.c line 30

Quoted 10 lines
kernel/scs.c new file +10 -0
+	 * task's thread_info::shadow_call_stack while that task is
+	 * running, and only save/restore the active shadow call stack
+	 * pointer when the usual register may be clobbered (e.g. across
+	 * context switches).
+	 *
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

kernel/scs.c line 40

Quoted 10 lines
kernel/scs.c new file +10 -0
+}
+
+static inline unsigned long *scs_magic(void *s)
+{
+	return (unsigned long *)(s + SCS_SIZE) - 1;
+}
+
+static inline void scs_set_magic(void *s)
+{
+	*scs_magic(s) = SCS_END_MAGIC;

You added task_set_scs() for this sort of thing, so I'm not convinced you need this extra helper.

kernel/scs.c line 46

Quoted 6 lines
kernel/scs.c new file +6 -0
+}
+
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+
+/* Matches NR_CACHED_STACKS for VMAP_STACK */
+#define NR_CACHED_SCS 2

Then they should probably both be derived from the same thing?

kernel/scs.c line 55

Quoted 9 lines
kernel/scs.c new file +9 -0
+static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
+
+static void *scs_alloc(int node)
+{
+	int i;
+	void *s;
+
+	for (i = 0; i < NR_CACHED_SCS; i++) {
+		s = this_cpu_xchg(scs_cache[i], NULL);

Might be worth a comment about the re-entrancy here.

Quoted 11 lines
kernel/scs.c new file +11 -0
+		if (s) {
+			memset(s, 0, SCS_SIZE);
+			goto out;
+		}
+	}
+
+	/*
+	 * We allocate a full page for the shadow stack, which should be
+	 * more than we need. Check the assumption nevertheless.
+	 */
+	BUILD_BUG_ON(SCS_SIZE > PAGE_SIZE);

kernel/scs.c line 76

Quoted 10 lines
kernel/scs.c new file +10 -0
+
+	s = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+				 VMALLOC_START, VMALLOC_END,
+				 GFP_SCS, PAGE_KERNEL, 0,
+				 node, __builtin_return_address(0));
+
+out:
+	if (s)
+		scs_set_magic(s);
+	/* TODO: poison for KASAN, unpoison in scs_free */

We don't usually commit these. What's missing?

Quoted 14 lines
kernel/scs.c new file +14 -0
+
+	return s;
+}
+
+static void scs_free(void *s)
+{
+	int i;
+
+	for (i = 0; i < NR_CACHED_SCS; i++)
+		if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
+			return;
+
+	vfree_atomic(s);
+}

kernel/scs.c line 100

Quoted 10 lines
kernel/scs.c new file +10 -0
+
+static int scs_cleanup(unsigned int cpu)
+{
+	int i;
+	void **cache = per_cpu_ptr(scs_cache, cpu);
+
+	for (i = 0; i < NR_CACHED_SCS; i++) {
+		vfree(cache[i]);
+		cache[i] = NULL;
+	}

Hmm, can this run concurrently with another CPU doing a stack allocation with this_cpu_cmpxchg()? It probably works out on arm64 thanks to the use of atomics, but we shouldn't be relying on that in core code.

Quoted 32 lines
kernel/scs.c new file +32 -0
+
+	return 0;
+}
+
+void __init scs_init(void)
+{
+	WARN_ON(cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
+			scs_cleanup) < 0);
+}
+
+#else /* !CONFIG_SHADOW_CALL_STACK_VMAP */
+
+static struct kmem_cache *scs_cache;
+
+static inline void *scs_alloc(int node)
+{
+	void *s;
+
+	s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
+	if (s) {
+		scs_set_magic(s);
+		/*
+		 * Poison the allocation to catch unintentional accesses to
+		 * the shadow stack when KASAN is enabled.
+		 */
+		kasan_poison_object_data(scs_cache, s);
+	}
+
+	return s;
+}
+
+static inline void scs_free(void *s)

kernel/scs.c line 142

Quoted 10 lines
kernel/scs.c new file +10 -0
+{
+	kasan_unpoison_object_data(scs_cache, s);
+	kmem_cache_free(scs_cache, s);
+}
+
+void __init scs_init(void)
+{
+	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
+				0, NULL);
+	WARN_ON(!scs_cache);

Memory allocation failure should be noisy enough without this.

kernel/scs.c line 154

Quoted 12 lines
kernel/scs.c new file +12 -0
+}
+
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+
+void scs_task_reset(struct task_struct *tsk)
+{
+	/*
+	 * Reset the shadow stack to the base address in case the task
+	 * is reused.
+	 */
+	task_set_scs(tsk, __scs_base(tsk));
+}

Why isn't this in the header?

Quoted 9 lines
kernel/scs.c new file +9 -0
+
+int scs_prepare(struct task_struct *tsk, int node)
+{
+	void *s;
+
+	s = scs_alloc(node);
+	if (!s)
+		return -ENOMEM;
+

kernel/scs.c line 173

Quoted 10 lines
kernel/scs.c new file +10 -0
+	task_set_scs(tsk, s);
+	return 0;
+}
+
+bool scs_corrupted(struct task_struct *tsk)
+{
+	unsigned long *magic = scs_magic(__scs_base(tsk));
+
+	return READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
+}

Same here.

kernel/scs.c line 185

Quoted 12 lines
kernel/scs.c new file +12 -0
+
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+
+	task_set_scs(tsk, NULL);

Aren't we about to free the task here? What does clearing the scs pointer achieve?

Will

#357 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 8 lines · Will Deacon

Makefile ifdef CONFIG_LIVEPATCH · line 873

Quoted 5 lines
Makefile +5 -0
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif

CFLAGS_SCS would seem more natural to me, although I see ftrace does it this way.

Right, I followed ftrace's example here.

Quoted 11 lines
Quoted 11 lines
arch/Kconfig +11 -0
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
Quoted 8 lines

arch/Kconfig config STACKPROTECTOR_STRONG · line 559

Quoted 6 lines
arch/Kconfig +6 -0
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.

Shouldn't some of this depend on CC_IS_CLANG?

Sure, I'll add CC_IS_CLANG here in the next version. Note that we do check for compiler support before selecting ARCH_SUPPORTS_SHADOW_CALL_STACK. The flags are architecture-specific, so the check is done in the arch Kconfig.

Quoted 4 lines
Quoted 4 lines
arch/Kconfig +4 -0
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
Quoted 8 lines

arch/Kconfig config STACKPROTECTOR_STRONG · line 567

Quoted 3 lines
arch/Kconfig +3 -0
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.

Given that this feature applies only to arm64 kernels built with clang, it feels weird to further segment that userbase with another config option. Does Android enable SHADOW_CALL_STACK_VMAP? If not, maybe we should ditch it for now and add it when we have a user.

Android doesn't enable the VMAP option right now due to increased memory overhead. I'll drop it from v12.

Quoted 8 lines

include/linux/scs.h line 29

Quoted 5 lines
include/linux/scs.h new file +5 -0
+/*
+ * A random number outside the kernel's virtual address space to mark the
+ * end of the shadow stack.
+ */
+#define SCS_END_MAGIC	0xaf0194819b1635f6UL

This seems like it might be arm64-specific. Why not choose something based off CONFIG_ILLEGAL_POINTER_VALUE (see linux/poison.h)?

Sure, I'll use POISON_POINTER_DELTA here.

Quoted 4 lines
Quoted 1 line

+static inline void *__scs_base(struct task_struct *tsk)

Please avoid using 'inline' in C files unless there's a good reason not to let the compiler figure it out.

Ack.

Quoted 5 lines

kernel/scs.c line 20

Quoted 3 lines
kernel/scs.c new file +3 -0
+{
+	/*
+	 * To minimize risk the of exposure, architectures may clear a

Should be "the risk of exposure".

Thanks.

Quoted 9 lines

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

Quoted 6 lines

kernel/scs.c line 40

Quoted 3 lines
kernel/scs.c new file +3 -0
+static inline void scs_set_magic(void *s)
+{
+	*scs_magic(s) = SCS_END_MAGIC;

You added task_set_scs() for this sort of thing, so I'm not convinced you need this extra helper.

Agreed, I'll drop this.

Quoted 5 lines

kernel/scs.c line 76

Quoted 3 lines
kernel/scs.c new file +3 -0
+	if (s)
+		scs_set_magic(s);
+	/* TODO: poison for KASAN, unpoison in scs_free */

We don't usually commit these. What's missing?

At the time, KASAN didn't support poisoning vmalloc'ed memory, but looks like that was fixed a while back.

Quoted 5 lines
Quoted 5 lines
kernel/scs.c new file +5 -0
+static int scs_cleanup(unsigned int cpu)
+{
+	int i;
+	void **cache = per_cpu_ptr(scs_cache, cpu);
+
Quoted 8 lines

kernel/scs.c line 100

Quoted 4 lines
kernel/scs.c new file +4 -0
+	for (i = 0; i < NR_CACHED_SCS; i++) {
+		vfree(cache[i]);
+		cache[i] = NULL;
+	}

Hmm, can this run concurrently with another CPU doing a stack allocation with this_cpu_cmpxchg()? It probably works out on arm64 thanks to the use of atomics, but we shouldn't be relying on that in core code.

This is essentially identical to the code in kernel/fork.c. Anyway, all of this code goes away with the VMAP option.

Quoted 7 lines

kernel/scs.c line 142

Quoted 5 lines
kernel/scs.c new file +5 -0
+void __init scs_init(void)
+{
+	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
+				0, NULL);
+	WARN_ON(!scs_cache);

Memory allocation failure should be noisy enough without this.

Sure, I'll remove the warning.

Quoted 11 lines

kernel/scs.c line 154

Quoted 8 lines
kernel/scs.c new file +8 -0
+void scs_task_reset(struct task_struct *tsk)
+{
+	/*
+	 * Reset the shadow stack to the base address in case the task
+	 * is reused.
+	 */
+	task_set_scs(tsk, __scs_base(tsk));
+}

Why isn't this in the header?

Quoted 8 lines

kernel/scs.c line 173

Quoted 6 lines
kernel/scs.c new file +6 -0
+bool scs_corrupted(struct task_struct *tsk)
+{
+	unsigned long *magic = scs_magic(__scs_base(tsk));
+
+	return READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
+}

Same here.

I'll move both to the header file.

Quoted 6 lines
Quoted 6 lines
kernel/scs.c new file +6 -0
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
Quoted 8 lines

kernel/scs.c line 185

Quoted 5 lines
kernel/scs.c new file +5 -0
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+
+	task_set_scs(tsk, NULL);

Aren't we about to free the task here? What does clearing the scs pointer achieve?

True, it doesn't achieve much, only leaves one fewer shadow stack pointer in memory. I'll drop this from the next version.

Sami

maintainer
#358 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.

Thanks,

Will

#359 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 20 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

Quoted 4 lines

But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.

It's a hardening step to just reduce the lifetime of a valid address exposed in memory. In fact, since there is a cache, I think it should be wiped even in scs_release().

Signature · 1 line
Kees Cook
maintainer
#360 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 21 lines · Kees Cook
Quoted 20 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

Quoted 8 lines
Quoted 4 lines

But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.

It's a hardening step to just reduce the lifetime of a valid address exposed in memory. In fact, since there is a cache, I think it should be wiped even in scs_release().

But we don't do this for /anything/ else and it forces alignment restrictions on the SCS allocation. Please either do it consistently, or not at all.

Will

#361 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

In reply to Will Deacon
Quoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 7 lines
Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move __scs_base() logic to cpu_switch_to() / scs_save().

Quoted 4 lines

But more generally, is it really worthwhile to do this clearing at all? Can you (or Kees?) provide some justification for it, please? We don't do it for anything else, e.g. the pointer authentication keys, so something feels amiss here.

Like Kees pointed out, this makes it slightly harder to locate the current task's shadow stack pointer. I realize there are other useful targets in thread_info, but we would rather not make this any easier than necessary. Is your primary concern here the cost of doing this, or just that it doesn't sufficiently improve security?

Sami

#362 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 21 lines · Sami Tolvanen
Quoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 7 lines
Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

Quoted 4 lines

I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move __scs_base() logic to cpu_switch_to() / scs_save().

Okay, so, I feel like this has gotten off into the weeds, or I'm really dense (or both). :) Going back to the original comment:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

I think there was some confusion about mixing the "we want to be able to wipe the value" combined with the masking in __scs_base(). These are unrelated, as was correctly observed with "We can still clear the base".

What I don't understand here is the suggestion to store two values:

Why is two better than storing one? With one, we only need a single access.

Why would storing the base be "friendlier on the allocations later on"? This is coming out of a single kmem cache, in 1K chunks. They will be naturally aligned to 1K (unless redzoing has been turned on for some slab debugging reason). The base masking is a way to avoid needing to store two values, and only happens at task death.

Storing two values eats memory for all tasks for seemingly no meaningful common benefit. What am I missing here?

Signature · 1 line
Kees Cook
maintainer
#363 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 30 lines · Kees Cook
Quoted 21 lines · Sami Tolvanen
Quoted 12 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen
Quoted 9 lines · Will Deacon

kernel/scs.c line 30

Quoted 5 lines
kernel/scs.c new file +5 -0
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

Quoted 7 lines
Quoted 2 lines

The idea is to avoid storing the current task's shadow stack address in memory, which is why I would rather not store the base address either.

What I mean is that, instead of storing the current shadow stack pointer, we instead store a base and an offset. We can still clear the base, as you do with the pointer today, and I don't see that the offset is useful to an attacker on its own.

Quoted 4 lines

I see what you mean. However, even if we store the base address + the offset, we still need aligned allocation if we want to clear the address. This would basically just move __scs_base() logic to cpu_switch_to() / scs_save().

Okay, so, I feel like this has gotten off into the weeds, or I'm really dense (or both). :) Going back to the original comment:

Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

Could we avoid forcing this alignment it we stored the SCS pointer as a (base,offset) pair instead? That might be friendlier on the allocations later on.

I think there was some confusion about mixing the "we want to be able to wipe the value" combined with the masking in __scs_base(). These are unrelated, as was correctly observed with "We can still clear the base".

Having just tried to implement this, it turns out they *are* related and we can't still clear the base, I was wrong about that :( See below.

Quoted 9 lines

What I don't understand here is the suggestion to store two values:

Why is two better than storing one? With one, we only need a single access.

Why would storing the base be "friendlier on the allocations later on"? This is coming out of a single kmem cache, in 1K chunks. They will be naturally aligned to 1K (unless redzoing has been turned on for some slab debugging reason). The base masking is a way to avoid needing to store two values, and only happens at task death.

Fair enough about the kmem_cache, although I'm still worried about these things getting bigger in future and the alignment having to increase at the same time. We also have a bunch of static/percpu allocations that don't use this cache.

Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS? The vmap version that I asked Sami to drop is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't pass VM_NO_GUARD. Looks like the same story for vmap stack :/

Quoted 2 lines

Storing two values eats memory for all tasks for seemingly no meaningful common benefit. What am I missing here?

I would like to remove the alignment requirements for the static and percpu allocations. AFAICT, the only reason the alignment is needed is because you want to convert an SCS pointer into the base pointer. The only reason *that* is needed is because of the questionable wiping of the pointer in the thread_info, but I really don't see the benefit of this. Unlike a crypto secret (which was your analogy), the SCS pointer is stored in memory in at least the following situations:

* The task isn't running
* The task is running in userspace
* The task is running a vCPU in KVM
* We're calling into EFI
* On exception entry from EL1, as part of stacking x18
* During CPU suspend

If we split the pointer in two (base, offset) then we could leave the base live in the thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.

The base and offset can live in the same cacheline and be loaded with ldp, so I don't see there being an access cost compared to a single variable.

Am I missing something (modulo us not agreeing on the utility of wiping the pointer)?

Will

#364 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 5 lines · Will Deacon

Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?

I agree that allocating from a kmem_cache isn't ideal for safety. It's a compromise to reduce memory overhead.

Quoted 4 lines

The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't pass VM_NO_GUARD. Looks like the same story for vmap stack :/

SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?

Quoted 5 lines

If we split the pointer in two (base, offset) then we could leave the base live in the thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.

I sent out v13 with split pointers, but I'm not sure it's convenient to add an overflow check to kernel_ventry where the VMAP_STACK check is done. I suppose I could add a check to kernel_entry after we load x18 from tsk. Thoughts?

Sami

maintainer
#365 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 9 lines · Sami Tolvanen
Quoted 5 lines · Will Deacon

Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?

I agree that allocating from a kmem_cache isn't ideal for safety. It's a compromise to reduce memory overhead.

Do you think it would be a problem if we always allocated a page for the SCS?

Quoted 7 lines
Quoted 4 lines

The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't pass VM_NO_GUARD. Looks like the same story for vmap stack :/

SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?

Sorry, I'd got the SCS upside-down in my head (hey, that second 'S' stands for 'Stack'!). But I think I'm right about vmap stack, which feels a little fragile even though it seems to work out today with the very limited uses of VM_NO_GUARD.

Quoted 10 lines
Quoted 5 lines

If we split the pointer in two (base, offset) then we could leave the base live in the thread_info, not require alignment of the stacks (which may allow for unconditional redzoning?) and then just update the offset value on context switch, which could be trivially checked as part of the existing stack overflow checking on kernel entry.

I sent out v13 with split pointers, but I'm not sure it's convenient to add an overflow check to kernel_ventry where the VMAP_STACK check is done. I suppose I could add a check to kernel_entry after we load x18 from tsk. Thoughts?

I'll take a look at v13, since at this stage I'm keen to get something queued up so that we can use it as a base for further improvements without you having to repost the whole stack every time.

Cheers,

Will

#366 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 6 lines · Will Deacon
Quoted 2 lines · Sami Tolvanen

I agree that allocating from a kmem_cache isn't ideal for safety. It's a compromise to reduce memory overhead.

Do you think it would be a problem if we always allocated a page for the SCS?

Yes, the memory overhead was deemed too large for Android devices, which have thousands of threads running.

Sami

#367 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

In reply to Will Deacon
Quoted 8 lines · Will Deacon
Quoted 7 lines · Sami Tolvanen
Quoted 5 lines · Will Deacon

Also, since you mentioned the lack of redzoning, isn't it a bit dodgy allocating blindly out of the kmem_cache? It means we don't have a redzone or a guard page, so if you can trigger something like a recursion bug then could you scribble past the SCS before the main stack overflows? Would this clobber somebody else's SCS?

Quoted 5 lines
Quoted 2 lines

I agree that allocating from a kmem_cache isn't ideal for safety. It's a compromise to reduce memory overhead.

Do you think it would be a problem if we always allocated a page for the SCS?

I guess doing this safely and without wasting a page per task would only be possible in an elegant way once MTE lands on devices?

I wonder how bad context switch latency would be if the actual SCS was percpu and vmapped (starting at an offset inside the page such that the SCS can only grow up to something like 0x400 bytes before panicking the CPU) and the context switch path saved/restored the used part of the vmapped SCS into a smaller allocation from the slab allocator... presumably the SCS will usually just be something like one cacheline big? That probably only costs a moderate amount of time to copy...
Or as an extension of that, if the SCS copying turns out to be too costly, there could be a percpu LRU cache consisting of vmapped SCS pages, and whenever a task gets scheduled that doesn't have a vmapped SCS, it "swaps out" the contents of the least recently used vmapped SCS into the corresponding task's slab SCS, and "swaps in" from its own slab SCS into the vmapped SCS. And task migration would force "swapping out".

Not sure if this is a good idea, or if I'm just making things worse by suggesting extra complexity...

#368 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

In reply to Will Deacon
Quoted 7 lines · Will Deacon
Quoted 6 lines · Sami Tolvanen
Quoted 4 lines · Will Deacon

The vmap version that I asked Sami to drop
is at least better in this regard, although the guard page is at the wrong end of the stack and we just hope that the allocation below us didn't pass VM_NO_GUARD. Looks like the same story for vmap stack :/

Quoted 7 lines
Quoted 2 lines

SCS grows up and the guard page is after the allocation, so how is it at the wrong end? Am I missing something here?

Sorry, I'd got the SCS upside-down in my head (hey, that second 'S' stands for 'Stack'!). But I think I'm right about vmap stack, which feels a little fragile even though it seems to work out today with the very limited uses of VM_NO_GUARD.

Yeah, when VMAP_STACK was originally being developed, IIRC, there was an effort made to eliminate all the users of VM_NO_GUARD, and it looks like it's mostly there. Really the only use left is arm64's kernel image mapping routines, and then it's not actually used in the traditional sense -- it's just a boolean for whether to toss in a guard page at the end of the data section, and the VMAs are built manually. I think that code could actually be refactored to drop it too and then the only user would be KASAN, which, IIUC, wants to build consecutive vmap areas.

Signature · 6 lines
Kees Cook

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
maintainer
#369 lore raw

Re: [PATCH v11 01/12] add support for Clang's Shadow Call Stack (SCS)

In reply to Will Deacon
Quoted 8 lines · On Mon, 20 Apr 2020 18:17:28 +0100 Will Deacon <[email protected]> wrote:

Makefile ifdef CONFIG_LIVEPATCH · line 873

Quoted 5 lines
Makefile +5 -0
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif  

CFLAGS_SCS would seem more natural to me, although I see ftrace does it this way.

The CC_FLAGS_FTRACE was added by Heiko Carstens, and the "CC_FLAGS_" appears to be a common usage in s390 :-)

That said, I like the CC_FLAGS_ notation, because the Linux build system uses CFLAGS_* as commands:

CFLAGS_foo.o = x
CFLAGS_REMOVE_foo.o = y

And "CC_FLAGS_" is only for new flags and easy to search for.

-- Steve

#370 lore raw

PATCH v11 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 10d7e818e118..502ab5447c8d 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 1b9de7d220fb..89aa96797743 100644
@@ -156,6 +156,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 69827d4fa052..721879d56bbd 100644
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 96d21a792b57..089602efa477 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer
#371 lore raw

Re: [PATCH v11 02/12] scs: add accounting

Quoted 28 lines · Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +3 -0
index 10d7e818e118..502ab5447c8d 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"

drivers/base/node.c static ssize_t node_read_meminfo(struct device *dev, · line 446

Quoted 6 lines
drivers/base/node.c +3 -0
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif

Why not just use KB everywhere instead of repeated division by 1024?

Will

#372 lore raw

Re: [PATCH v11 02/12] scs: add accounting

Quoted 5 lines · Will Deacon

drivers/base/node.c static ssize_t node_read_meminfo(struct device *dev, · line 446

Quoted 3 lines
drivers/base/node.c +3 -0
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif

Why not just use KB everywhere instead of repeated division by 1024?

This was to correctly calculate memory usage with shadow stacks <1024 bytes. I don't think we need that anymore, so I'll change this to _KB in the next version.

Sami

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#373 lore raw

PATCH v11 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer
#374 lore raw

Re: [PATCH v11 03/12] scs: add support for stack usage debugging

Quoted 16 lines · Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +0 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 195

Quoted 10 lines
kernel/scs.c +9 -0
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;

I think the expectation is that the caller has already checked that the stack is not corrupted, so I'd probably throw a couple of underscores in front of the function name, along with a comment.

Also, is tsk ever != current?

Quoted 12 lines
kernel/scs.c +12 -0
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 217

Quoted 10 lines
kernel/scs.c +10 -0
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);

Do you really need this lock? I'd have thought you could cmpxchg() highest instead.

Will

#375 lore raw

Re: [PATCH v11 03/12] scs: add support for stack usage debugging

Quoted 5 lines · Will Deacon
Quoted 5 lines
kernel/scs.c +5 -0
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
Quoted 8 lines

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 195

Quoted 4 lines
kernel/scs.c +4 -0
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;

I think the expectation is that the caller has already checked that the stack is not corrupted, so I'd probably throw a couple of underscores in front of the function name, along with a comment.

Correct. I'll do that.

Quoted 1 line

Also, is tsk ever != current?

This is only called from scs_release(), so tsk is never current.

Quoted 13 lines
Quoted 13 lines
kernel/scs.c +13 -0
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
Quoted 8 lines

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 217

Quoted 5 lines
kernel/scs.c +5 -0
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);

Do you really need this lock? I'd have thought you could cmpxchg() highest instead.

This is similar to check_stack_usage in kernel/exit.c, but yes, I can change this to a cmpxchg() loop instead.

Sami

#376 lore raw

PATCH v11 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 691a552c2cc3..c53cb9025ad2 100644
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer
#377 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

Quoted 15 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Quoted 8 lines

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

SCS would actually provide another way to do return hooking. An arguably much saner model at that.

The 'normal' way is to (temporary) replace the on-stack return value, and then replace it back in the return handler. This is because we can't simply push a fake return on the stack, because that would wreck the expected stack layout of the regular function.

But there is nothing that would stop us from pushing an extra entry on the SCS. It would in fact be a much cleaner solution. The entry hook sticks an extra entry on the SCS, the function ignores what's on the normal stack and pops from the SCS, we return to the exit handler, which in turn pops from the SCS stack at which point we're back to regular.

The only 'funny' is that the exit handler itself should not push to the SCS, or we should frob the return-to-exit-handler such that it lands after the push.

Quoted 12 lines
---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +0 -0
index 691a552c2cc3..c53cb9025ad2 100644

arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548

Quoted 7 lines
arch/Kconfig +1 -0
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a

AFAICT you also need to kill KRETPROBES, which plays similar games. And doesn't BPF also do stuff like this?

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

reviewer
#378 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

Hi Peter,

Quoted 33 lines · Peter Zijlstra
Quoted 15 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Quoted 8 lines

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

SCS would actually provide another way to do return hooking. An arguably much saner model at that.

The 'normal' way is to (temporary) replace the on-stack return value, and then replace it back in the return handler. This is because we can't simply push a fake return on the stack, because that would wreck the expected stack layout of the regular function.

Quoted 5 lines

But there is nothing that would stop us from pushing an extra entry on the SCS. It would in fact be a much cleaner solution. The entry hook sticks an extra entry on the SCS, the function ignores what's on the normal stack and pops from the SCS, we return to the exit handler, which in turn pops from the SCS stack at which point we're back to regular.

For background: on arm64 we wanted to use DYNAMIC_FTRACE_WITH_REGS since we already have to use that to handle pointer authentication, and didn't want to gain more ways of implementing ftrace.

Arguably we should move the dependency into the arm64 Kconfig for ARCH_SUPPORTS_SHADOW_CALL_STACK.

Quoted 17 lines

The only 'funny' is that the exit handler itself should not push to the SCS, or we should frob the return-to-exit-handler such that it lands after the push.

Quoted 13 lines
---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +0 -0
index 691a552c2cc3..c53cb9025ad2 100644
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
Quoted 8 lines

arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548

Quoted 6 lines
arch/Kconfig +1 -0
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a

AFAICT you also need to kill KRETPROBES, which plays similar games.

Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...

Quoted 1 line

And doesn't BPF also do stuff like this?

Can BPF mess with return addresses now!?

Thanks,
Mark.

maintainer
#379 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

Quoted 15 lines · Mark Rutland
Quoted 6 lines
Quoted 6 lines
arch/Kconfig +0 -0
index 691a552c2cc3..c53cb9025ad2 100644
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
Quoted 8 lines

arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548

Quoted 6 lines
arch/Kconfig +1 -0
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a

AFAICT you also need to kill KRETPROBES, which plays similar games.

Quoted 5 lines

Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...

To be fair, I've not looked at the arm64 implementation. x86 does gross things like ftrace does. On x86 ftrace_graph and kretprobe also can't be on at the same time for the same function, there's some yuck around there.

Rostedt was recently talking about cleaning some of that up.

But if kretprobe can work on arm64, then ftrace_graph can too, but I think that links back to what you said earlier, you didn't want more ftrace variants or something.

Quoted 3 lines
Quoted 1 line

And doesn't BPF also do stuff like this?

Can BPF mess with return addresses now!?

At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

reviewer
#380 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

Quoted 22 lines · Peter Zijlstra
Quoted 15 lines · Mark Rutland
Quoted 6 lines
Quoted 6 lines
arch/Kconfig +0 -0
index 691a552c2cc3..c53cb9025ad2 100644
@@ -542,6 +542,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
Quoted 8 lines

arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 548

Quoted 6 lines
arch/Kconfig +1 -0
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a

AFAICT you also need to kill KRETPROBES, which plays similar games.

Quoted 5 lines

Hmm... how does KREPROBES work? If you can only mess with the return address when probing the first instruction in the function, it'll just work for SCS or pointer authentication, as the LR is used at that instant. If KRETPROBES tries to mess with the return address elsewhere it'd be broken today...

Quoted 4 lines

To be fair, I've not looked at the arm64 implementation. x86 does gross things like ftrace does. On x86 ftrace_graph and kretprobe also can't be on at the same time for the same function, there's some yuck around there.

I can imagine the same holds true for us there.

Quoted 5 lines

Rostedt was recently talking about cleaning some of that up.

But if kretprobe can work on arm64, then ftrace_graph can too, but I think that links back to what you said earlier, you didn't want more ftrace variants or something.

I just want to avoid yet another implementation of the underlying mechanism. For DYNAMIC_FTRACE_WITH_REGS we can mess with the LR before pauth or SCS sees it, so those definitely work.

If KRETPROBES works by messing with the LR at the instnat the function is entered, that should work similarly. If it works by replacing the RET it should also work out since any pauth/SCS work will have been undone by that point. If it attempts to mess with the return address in the middle of a function then it's not reliable today.

I'll take a look, since

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

And doesn't BPF also do stuff like this?

Can BPF mess with return addresses now!?

At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.

Sounds like I might need to do some digging...

Mark.

#381 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

Quoted 5 lines · Mark Rutland

If KRETPROBES works by messing with the LR at the instnat the function is entered, that should work similarly. If it works by replacing the RET it should also work out since any pauth/SCS work will have been undone by that point. If it attempts to mess with the return address in the middle of a function then it's not reliable today.

I did initially have a patch to disable kretprobes (until v5), but as Mark pointed out back then, the return address is modified before it gets pushed to the shadow stack, so there was no conflict with SCS. I confirmed this on arm64, but haven't looked at other architectures.

Sami

maintainer
#382 lore raw

Re: [PATCH v11 04/12] scs: disable when function graph tracing is enabled

In reply to Mark Rutland
Quoted 8 lines · On Fri, 17 Apr 2020 16:46:14 +0100 Mark Rutland <[email protected]> wrote:
Quoted 6 lines
Quoted 3 lines
Quoted 1 line

And doesn't BPF also do stuff like this?

Can BPF mess with return addresses now!?

At least on x86 I think it does. But what do I know, I can't operate that stuff. Rostedt might know.

Sounds like I might need to do some digging...

May want to ping Alexei. It appears that if BPF adds a direct hook to a function, it will prevent the function graph tracer from tracing it. :-/

-- Steve

#383 lore raw

PATCH v11 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 85e4149cc5d5..409a6c1be8cc 100644
@@ -81,6 +81,10 @@ endif
 
 KBUILD_CFLAGS += $(branch-prot-flags-y)
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#384 lore raw

PATCH v11 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index 197a9ba2d5ea..ed15be0f8103 100644
@@ -58,6 +58,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#385 lore raw

PATCH v11 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#386 lore raw

PATCH v11 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#387 lore raw

PATCH v11 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index fe57f60f06a8..875b106c5d98 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#388 lore raw

PATCH v11 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 33 +++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  8 ++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 40fb05d96c60..c380a16533f6 100644
@@ -64,6 +64,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index 512174a8e789..1fb651f73da3 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index 4e5b8ee31442..151f28521f1e 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index 9981a0a5a87f..777a662888ec 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
index ddcde093c433..c33264ce7258 100644
@@ -179,6 +179,11 @@ alternative_cb_end
 	apply_ssbd 1, x22, x23
 
 	ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to)
 	mov	sp, x9
 	msr	sp_el0, x1
 	ptrauth_keys_install_kernel x1, 1, x8, x9, x10
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 SYM_FUNC_END(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
index 57a91032b4c2..1514445bbccb 100644
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 56be4cbf771f..a35d3318492c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	entry_task_switch(next);
 	uao_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index 061f60fe452f..1d112e34a636 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)
 	unsigned int cpu = smp_processor_id();
 	const struct cpu_operations *ops = get_cpu_ops(cpu);
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#389 lore raw

PATCH v11 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index c33264ce7258..768cd7abd32c 100644
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#390 lore raw

PATCH v11 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 094eabdecfe6..b52ae8c29560 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+# remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#391 lore raw

v12 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v12:
- Removed CONFIG_SHADOW_CALL_STACK_VMAP.
- Added CC_IS_CLANG as a dependency to CONFIG_SHADOW_CALL_STACK.
- Changed SCS_END_MAGIC to use POISON_POINTER_DELTA.
- Removed the unnecessary scs_set_magic() helper function.
- Moved scs_task_reset() and scs_corrupted() to scs.h, along with __scs_magic() and __scs_base().
- Removed a redundant warning from memory allocation.
- Removed an unnecessary task_set_scs() call from scs_release().
- Changed the accounting code to calculate KiB instead of bytes.
- Replaced the lock in scs_check_usage() with a cmpxchg() loop.

Changes in v11:
- Rebased, added maintainers for kernel/ changes.

Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.

Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 ++
 arch/Kconfig                          |  26 ++++++
 arch/arm64/Kconfig                    |   5 ++
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  34 ++++++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |   3 +
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 ++-
 arch/arm64/kernel/entry.S             |  47 +++++++++-
 arch/arm64/kernel/head.S              |   8 ++
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               |  21 +++++
 arch/arm64/kernel/smp.c               |   4 +
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 +++
 drivers/base/node.c                   |   6 ++
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 ++
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  92 ++++++++++++++++++++
 init/init_task.c                      |   8 ++
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 ++
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 121 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 ++
 mm/vmstat.c                           |   3 +
 32 files changed, 456 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: ae83d0b416db002fe95601e7f97f64b59514d936

Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#392 lore raw

PATCH v12 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |  6 +++
 arch/Kconfig                   | 25 +++++++++
 include/linux/compiler-clang.h |  6 +++
 include/linux/compiler_types.h |  4 ++
 include/linux/scs.h            | 92 ++++++++++++++++++++++++++++++++++
 init/init_task.c               |  8 +++
 kernel/Makefile                |  1 +
 kernel/fork.c                  |  9 ++++
 kernel/sched/core.c            |  2 +
 kernel/scs.c                   | 67 +++++++++++++++++++++++++
 10 files changed, 220 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 49b2709ff44e..6094db2c7252 100644
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +25 -0
index 786a85d4ad40..8450d56e6af6 100644
@@ -533,6 +533,31 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +92 -0

92 lines not on this page · raw message

init/init_task.c +8 -0
index bd403ed3e418..aaa71366d162 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 8c700f881d92..f6339f9d232d 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 3a61a3b8eaa9..c99620c1ec20 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +67 -0

67 lines not on this page · raw message

Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#393 lore raw

Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 371 lines · Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |  6 +++
 arch/Kconfig                   | 25 +++++++++
 include/linux/compiler-clang.h |  6 +++
 include/linux/compiler_types.h |  4 ++
 include/linux/scs.h            | 92 ++++++++++++++++++++++++++++++++++
 init/init_task.c               |  8 +++
 kernel/Makefile                |  1 +
 kernel/fork.c                  |  9 ++++
 kernel/sched/core.c            |  2 +
 kernel/scs.c                   | 67 +++++++++++++++++++++++++
 10 files changed, 220 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 49b2709ff44e..6094db2c7252 100644
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +25 -0
index 786a85d4ad40..8450d56e6af6 100644
@@ -533,6 +533,31 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +92 -0
new file mode 100644
index 000000000000..051d27ad3da4
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#ifndef _LINUX_SCS_H
+#define _LINUX_SCS_H
+
+#include <linux/gfp.h>
+#include <linux/poison.h>
+#include <linux/sched.h>
+#include <asm/page.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+/*
+ * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
+ * architecture) provided ~40% safety margin on stack usage while keeping
+ * memory allocation overhead reasonable.
+ */
+#define SCS_SIZE	1024UL
+#define GFP_SCS		(GFP_KERNEL | __GFP_ZERO)
+
+/* An illegal pointer value to mark the end of the shadow stack. */
+#define SCS_END_MAGIC	(0x5f6UL + POISON_POINTER_DELTA)
+
+#define task_scs(tsk)	(task_thread_info(tsk)->shadow_call_stack)
+
+static inline void task_set_scs(struct task_struct *tsk, void *s)
+{
+	task_scs(tsk) = s;
+}
+
+extern void scs_init(void);
+
+static inline void *__scs_base(struct task_struct *tsk)
+{
+	/*
+	 * To minimize the risk of exposure, architectures may clear a
+	 * task's thread_info::shadow_call_stack while that task is
+	 * running, and only save/restore the active shadow call stack
+	 * pointer when the usual register may be clobbered (e.g. across
+	 * context switches).
+	 *
+	 * The shadow call stack is aligned to SCS_SIZE, and grows
+	 * upwards, so we can mask out the low bits to extract the base
+	 * when the task is not running.
+	 */
+	return (void *)((unsigned long)task_scs(tsk) & ~(SCS_SIZE - 1));
+}
+
+static inline void scs_task_reset(struct task_struct *tsk)
+{
+	/*
+	 * Reset the shadow stack to the base address in case the task
+	 * is reused.
+	 */
+	task_set_scs(tsk, __scs_base(tsk));
+}
+
+extern int scs_prepare(struct task_struct *tsk, int node);
+
+static inline unsigned long *__scs_magic(void *s)
+{
+	return (unsigned long *)(s + SCS_SIZE) - 1;
+}
+
+static inline bool scs_corrupted(struct task_struct *tsk)
+{
+	unsigned long *magic = __scs_magic(__scs_base(tsk));
+
+	return READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
+}
+
+extern void scs_release(struct task_struct *tsk);
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+#define task_scs(tsk)	NULL
+
+static inline void task_set_scs(struct task_struct *tsk, void *s) {}
+static inline void scs_init(void) {}
+static inline void scs_task_reset(struct task_struct *tsk) {}
+static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
+static inline bool scs_corrupted(struct task_struct *tsk) { return false; }
+static inline void scs_release(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* _LINUX_SCS_H */
init/init_task.c +8 -0
index bd403ed3e418..aaa71366d162 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -185,6 +186,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 8c700f881d92..f6339f9d232d 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 3a61a3b8eaa9..c99620c1ec20 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6045,6 +6046,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +55 -0
new file mode 100644
index 000000000000..e1a8fc453b86
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/kasan.h>
+#include <linux/scs.h>
+#include <linux/slab.h>
+#include <asm/scs.h>
+
+static struct kmem_cache *scs_cache;
+
+static void *scs_alloc(int node)
+{
+	void *s;
+
+	s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
+	if (s) {
+		*__scs_magic(s) = SCS_END_MAGIC;
+		/*
+		 * Poison the allocation to catch unintentional accesses to
+		 * the shadow stack when KASAN is enabled.
+		 */
+		kasan_poison_object_data(scs_cache, s);
+	}
+
+	return s;
+}
+
+static void scs_free(void *s)
+{
+	kasan_unpoison_object_data(scs_cache, s);
+	kmem_cache_free(scs_cache, s);
+}
+
+void __init scs_init(void)
+{
+	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
+				0, NULL);
+}
+
+int scs_prepare(struct task_struct *tsk, int node)
+{
+	void *s;
+
+	s = scs_alloc(node);
+	if (!s)
+		return -ENOMEM;
+
+	task_set_scs(tsk, s);
+	return 0;
+}
+

kernel/scs.c line 65

Quoted 10 lines
kernel/scs.c new file +10 -0
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+

I'd like to have task_set_scs(tsk, NULL) retained here, to avoid need to depend on the released task memory getting scrubbed at a later time. Exposures of this pointer would expose the region of other SCSs.

-Kees

Quoted 5 lines
kernel/scs.c new file +2 -0
+	scs_free(s);
+}
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
Signature · 1 line
Kees Cook
maintainer
#394 lore raw

Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 14 lines · Kees Cook

kernel/scs.c line 65

Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10 -0
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+

I'd like to have task_set_scs(tsk, NULL) retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.

Hmm, doesn't it get zeroed almost immediately by kmem_cache_free() if INIT_ON_FREE_DEFAULT_ON is set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in the task_struct and treating this specially feels odd to me.

Will

#395 lore raw

Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 16 lines · Will Deacon
Quoted 14 lines · Kees Cook

kernel/scs.c line 65

Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10 -0
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+

I'd like to have task_set_scs(tsk, NULL) retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.

Quoted 4 lines

Hmm, doesn't it get zeroed almost immediately by kmem_cache_free() if INIT_ON_FREE_DEFAULT_ON is set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in the task_struct and treating this specially feels odd to me.

That's going to be an uncommon config except for the most paranoid of system builders. :) Having this get wiped particular thing wiped is just a decent best practice for what is otherwise treated as a "secret", just like crypto routines wipe their secrets before free().

Signature · 1 line
Kees Cook
maintainer
#396 lore raw

Re: [PATCH v12 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 17 lines · Kees Cook
Quoted 16 lines · Will Deacon
Quoted 14 lines · Kees Cook

kernel/scs.c line 65

Quoted 10 lines · Sami Tolvanen
kernel/scs.c new file +10 -0
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+

I'd like to have task_set_scs(tsk, NULL) retained here, to avoid need to depend on the released task memory getting scrubbed at a later time.

Quoted 7 lines
Quoted 4 lines

Hmm, doesn't it get zeroed almost immediately by kmem_cache_free() if INIT_ON_FREE_DEFAULT_ON is set? That seems much better than special-casing SCS, as there's a tonne of other useful stuff kicking around in the task_struct and treating this specially feels odd to me.

That's going to be an uncommon config except for the most paranoid of system builders. :)

Sounds like a perfect fit, then ;)

Quoted 3 lines

Having this get wiped particular thing wiped is just
a decent best practice for what is otherwise treated as a "secret", just like crypto routines wipe their secrets before free().

Sorry, but I don't buy that analogy. The SCS pointer is stored in memory all over the place and if it needs to treated in the same way as crypto secrets then this whole thing needs rethinking. On top of that, where crypto routines may wipe their secrets, we don't do what is being proposed for the SCS pointer to other similar pieces of data, such as pointer authentication keys.

Will

#397 lore raw

PATCH v12 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 16 ++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 38 insertions(+)
drivers/base/node.c +6 -0
index 10d7e818e118..50b8c0d43859 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_KB),
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..09cd51c8d23d 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_KB));
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 1b9de7d220fb..acffc3bc6178 100644
@@ -156,6 +156,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_KB,	/* measured in KiB */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +16 -0
index e1a8fc453b86..7eea2d97bd2d 100644
@@ -6,8 +6,10 @@
  */
 
 #include <linux/kasan.h>
+#include <linux/mm.h>
 #include <linux/scs.h>
 #include <linux/slab.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static struct kmem_cache *scs_cache;
@@ -41,6 +43,17 @@ void __init scs_init(void)
 				0, NULL);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_KB,
+		account * (SCS_SIZE / 1024));
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -50,6 +63,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -63,5 +78,6 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 69827d4fa052..83743d7a6177 100644
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_KB),
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 96d21a792b57..2435d2c24657 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer Acked-by on v12 2/12
#398 lore raw

Re: [PATCH v12 02/12] scs: add accounting

Quoted 4 lines · Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

Quoted 8 lines
---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 16 ++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 38 insertions(+)

Thanks!

Will

#399 lore raw

PATCH v12 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
kernel/scs.c +38 -0
index 7eea2d97bd2d..147917e31adf 100644
@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static unsigned long __scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = __scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static unsigned long highest;
+	unsigned long used = __scs_used(tsk);
+	unsigned long prev;
+	unsigned long curr = highest;
+
+	while (used > curr) {
+		prev = cmpxchg(&highest, curr, used);
+
+		if (prev == curr) {
+			pr_info("%s (%d): highest shadow stack usage: "
+				"%lu bytes\n",
+				tsk->comm, task_pid_nr(tsk), used);
+			break;
+		}
+
+		curr = prev;
+	}
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk) {}
+#endif
+
 void scs_release(struct task_struct *tsk)
 {
 	void *s;
@@ -77,6 +114,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	scs_free(s);
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
maintainer Acked-by on v12 3/12
#400 lore raw

Re: [PATCH v12 03/12] scs: add support for stack usage debugging

Quoted 29 lines · Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
kernel/scs.c +12 -0
index 7eea2d97bd2d..147917e31adf 100644
@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static unsigned long __scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = __scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 92

Quoted 10 lines
kernel/scs.c +10 -0
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static unsigned long highest;
+	unsigned long used = __scs_used(tsk);
+	unsigned long prev;
+	unsigned long curr = highest;
+
+	while (used > curr) {
+		prev = cmpxchg(&highest, curr, used);

I think this can be cmpxchg_relaxed(), since we don't care about ordering here afaict.

With that:

Cheers,

Will

#401 lore raw

Re: [PATCH v12 03/12] scs: add support for stack usage debugging

Quoted 4 lines · Will Deacon
Quoted 4 lines
kernel/scs.c +4 -0
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static unsigned long highest;
+	unsigned long used = __scs_used(tsk);
Quoted 8 lines

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 92

Quoted 5 lines
kernel/scs.c +5 -0
+	unsigned long prev;
+	unsigned long curr = highest;
+
+	while (used > curr) {
+		prev = cmpxchg(&highest, curr, used);

I think this can be cmpxchg_relaxed(), since we don't care about ordering here afaict.

Sure, I'll change this in v13. Thanks.

Sami

#402 lore raw

PATCH v12 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 8450d56e6af6..b52929f38cf7 100644
@@ -543,6 +543,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
 	depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
 	  shadow stack to protect function return addresses from being
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#403 lore raw

PATCH v12 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 85e4149cc5d5..409a6c1be8cc 100644
@@ -81,6 +81,10 @@ endif
 
 KBUILD_CFLAGS += $(branch-prot-flags-y)
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#404 lore raw

PATCH v12 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index 197a9ba2d5ea..ed15be0f8103 100644
@@ -58,6 +58,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#405 lore raw

PATCH v12 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#406 lore raw

PATCH v12 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#407 lore raw

PATCH v12 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index fe57f60f06a8..875b106c5d98 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#408 lore raw

PATCH v12 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 34 ++++++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 33 +++++++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  8 +++++++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 16 +++++++++++++
 arch/arm64/kernel/smp.c              |  4 ++++
 10 files changed, 107 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 40fb05d96c60..c380a16533f6 100644
@@ -64,6 +64,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +34 -0
new file mode 100644
index 000000000000..27dced9cbaf7
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index 512174a8e789..1fb651f73da3 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index 4e5b8ee31442..151f28521f1e 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index 9981a0a5a87f..777a662888ec 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +31 -2
index ddcde093c433..14f0ff763b39 100644
@@ -179,6 +179,11 @@ alternative_cb_end
 	apply_ssbd 1, x22, x23
 
 	ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -280,6 +285,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -388,6 +399,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -405,15 +419,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	adr_this_cpu x18, irq_shadow_call_stack, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -901,6 +925,11 @@ SYM_FUNC_START(cpu_switch_to)
 	mov	sp, x9
 	msr	sp_el0, x1
 	ptrauth_keys_install_kernel x1, 1, x8, x9, x10
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 SYM_FUNC_END(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +8 -0
index 57a91032b4c2..1514445bbccb 100644
@@ -424,6 +424,10 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -737,6 +741,10 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/process.c +2 -0
index 56be4cbf771f..a35d3318492c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	entry_task_switch(next);
 	uao_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +16 -0
new file mode 100644
index 000000000000..086ad97bba86
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <asm/scs.h>
+
+/* Allocate a static per-CPU shadow stack */
+#define DEFINE_SCS(name)						\
+	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
+		__aligned(SCS_SIZE)
+
+DEFINE_SCS(irq_shadow_call_stack);
arch/arm64/kernel/smp.c +4 -0
index 061f60fe452f..1d112e34a636 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -370,6 +371,9 @@ void cpu_die(void)
 	unsigned int cpu = smp_processor_id();
 	const struct cpu_operations *ops = get_cpu_ops(cpu);
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#409 lore raw

PATCH v12 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/kernel/entry.S | 14 +++++++++++++-
 arch/arm64/kernel/scs.c   |  5 +++++
 2 files changed, 18 insertions(+), 1 deletion(-)
arch/arm64/kernel/entry.S +13 -1
index 14f0ff763b39..9f7be489d26d 100644
@@ -1058,13 +1058,16 @@ SYM_CODE_START(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1074,6 +1077,15 @@ SYM_CODE_START(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal, tmp=x6
+	b	4f
+3:	adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +5 -0
index 086ad97bba86..656262736eca 100644
@@ -14,3 +14,8 @@
 		__aligned(SCS_SIZE)
 
 DEFINE_SCS(irq_shadow_call_stack);
+
+#ifdef CONFIG_ARM_SDE_INTERFACE
+DEFINE_SCS(sdei_shadow_call_stack_normal);
+DEFINE_SCS(sdei_shadow_call_stack_critical);
+#endif
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#410 lore raw

PATCH v12 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 094eabdecfe6..b52ae8c29560 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+# remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.1.301.g55bc3eb7cb9-goog
#411 lore raw

v13 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v13:
- Changed thread_info::shadow_call_stack to a base address and an offset instead, and removed the now unneeded __scs_base() and scs_save().
- Removed alignment from the kmem_cache and static allocations.
- Removed the task_set_scs() helper function.
- Moved the assembly code for loading and storing the offset in thread_info to scs_load/save macros.
- Added offset checking to scs_corrupted().
- Switched to cmpxchg_relaxed() in scs_check_usage().

Changes in v12:
- Removed CONFIG_SHADOW_CALL_STACK_VMAP.
- Added CC_IS_CLANG as a dependency to CONFIG_SHADOW_CALL_STACK.
- Changed SCS_END_MAGIC to use POISON_POINTER_DELTA.
- Removed the unnecessary scs_set_magic() helper function.
- Moved scs_task_reset() and scs_corrupted() to scs.h, along with __scs_magic() and __scs_base().
- Removed a redundant warning from memory allocation.
- Removed an unnecessary task_set_scs() call from scs_release().
- Changed the accounting code to calculate KiB instead of bytes.
- Replaced the lock in scs_check_usage() with a cmpxchg() loop.

Changes in v11:
- Rebased, added maintainers for kernel/ changes.

Changes in v10:
- Removed an unnecessary <asm/scs.h> include from head.S.

Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 ++
 arch/Kconfig                          |  25 ++++++
 arch/arm64/Kconfig                    |   5 ++
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  46 ++++++++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |  13 +++
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   4 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 ++-
 arch/arm64/kernel/entry.S             |  38 +++++++-
 arch/arm64/kernel/head.S              |   6 ++
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               |  20 +++++
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 +++
 drivers/base/node.c                   |   6 ++
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 ++
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  72 +++++++++++++++
 init/init_task.c                      |   8 ++
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 ++
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 121 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 ++
 mm/vmstat.c                           |   3 +
 31 files changed, 442 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 6a8b55ed4056ea5559ebe4f6a4b247f627870d4c

Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#412 lore raw

PATCH v13 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the stacks.

---
 Makefile                       |  6 +++
 arch/Kconfig                   | 24 ++++++++++++
 include/linux/compiler-clang.h |  6 +++
 include/linux/compiler_types.h |  4 ++
 include/linux/scs.h            | 72 ++++++++++++++++++++++++++++++++++
 init/init_task.c               |  8 ++++
 kernel/Makefile                |  1 +
 kernel/fork.c                  |  9 +++++
 kernel/sched/core.c            |  2 +
 kernel/scs.c                   | 68 ++++++++++++++++++++++++++++++++
 10 files changed, 200 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 679f302a8b8b..33dc0d0cdd08 100644
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +24 -0
index 786a85d4ad40..334a3d9b19df 100644
@@ -533,6 +533,30 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the
+	  ones documented for user space. The kernel must store addresses
+	  of shadow stacks in memory, which means an attacker capable of
+	  reading and writing arbitrary memory may be able to locate them
+	  and hijack control flow by modifying the stacks.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +72 -0

72 lines not on this page · raw message

init/init_task.c +8 -0
index bd403ed3e418..169e34066d35 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -50,6 +51,13 @@ static struct sighand_struct init_sighand = {
 	.signalfd_wqh	= __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh),
 };
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)]
+		__init_task_data = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Set up the first task table, touch at your own risk!. Base=0,
  * limit=0x1fffff (=2MB)
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 8c700f881d92..f6339f9d232d 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 9a2fbf98fd6f..934e03cfaec7 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6040,6 +6041,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +68 -0

68 lines not on this page · raw message

Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#413 lore raw

Re: [PATCH v13 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 6 lines · Sami Tolvanen
include/linux/compiler-clang.h +0 -0
index 333a6695a918..18fc4d29ef27 100644

include/linux/compiler-clang.h line 50

Quoted 9 lines
include/linux/compiler-clang.h +6 -0
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs       __attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif

Can we remove the #else branch? compiler_types.h [*] has to care anyway about that case for other compilers anyway, no?

include/linux/compiler_types.h struct ftrace_likely_data { · line 198

Quoted 11 lines
include/linux/compiler_types.h +3 -0
index e970f97a7fcb..97b62f47a80d 100644
@@ -193,6 +193,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif

[*] Here

Cheers,
Miguel

#414 lore raw

Re: [PATCH v13 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 11 lines · Miguel Ojeda
Quoted 10 lines · Sami Tolvanen
include/linux/compiler-clang.h +1 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
Quoted 8 lines

include/linux/compiler-clang.h line 50

Quoted 5 lines
include/linux/compiler-clang.h +5 -0
+#if __has_feature(shadow_call_stack)
+# define __noscs       __attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif

Can we remove the #else branch? compiler_types.h [*] has to care anyway about that case for other compilers anyway, no?

Yes, it's unnecessary. I'll remove this in the next version. Thanks!

Sami

#415 lore raw

PATCH v13 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 15 +++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 37 insertions(+)
drivers/base/node.c +6 -0
index 10d7e818e118..50b8c0d43859 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_KB),
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..09cd51c8d23d 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_KB));
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 1b9de7d220fb..acffc3bc6178 100644
@@ -156,6 +156,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_KB,	/* measured in KiB */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +15 -0
index 43624be9ad90..8769016c714c 100644
@@ -6,8 +6,10 @@
  */
 
 #include <linux/kasan.h>
+#include <linux/mm.h>
 #include <linux/scs.h>
 #include <linux/slab.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static struct kmem_cache *scs_cache;
@@ -40,6 +42,17 @@ void __init scs_init(void)
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(task_scs(tsk));
+}
+
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_KB,
+		account * (SCS_SIZE / 1024));
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -50,6 +63,7 @@ int scs_prepare(struct task_struct *tsk, int node)
 
 	task_scs(tsk) = s;
 	task_scs_offset(tsk) = 0;
+	scs_account(tsk, 1);
 
 	return 0;
 }
@@ -64,5 +78,6 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 69827d4fa052..83743d7a6177 100644
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_KB),
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 96d21a792b57..2435d2c24657 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#416 lore raw

PATCH v13 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
kernel/scs.c +38 -0
index 8769016c714c..2a96573f2b1b 100644
@@ -68,6 +68,43 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static unsigned long __scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = task_scs(tsk);
+	unsigned long *end = __scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static unsigned long highest;
+	unsigned long used = __scs_used(tsk);
+	unsigned long prev;
+	unsigned long curr = highest;
+
+	while (used > curr) {
+		prev = cmpxchg_relaxed(&highest, curr, used);
+
+		if (prev == curr) {
+			pr_info("%s (%d): highest shadow stack usage: "
+				"%lu bytes\n",
+				tsk->comm, task_pid_nr(tsk), used);
+			break;
+		}
+
+		curr = prev;
+	}
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk) {}
+#endif
+
 void scs_release(struct task_struct *tsk)
 {
 	void *s;
@@ -77,6 +114,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	scs_free(s);
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#417 lore raw

PATCH v13 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

With SCS the return address is taken from the shadow stack and the value in the frame record has no effect. The mcount based graph tracer hooks returns by modifying frame records on the (regular) stack, and thus is not compatible. The patchable-function-entry graph tracer used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved to the shadow stack, and is compatible.

Modifying the mcount based graph tracer to work with SCS would require a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), and we expect that everyone will eventually move to the patchable-function-entry based graph tracer anyway, so for now let's disable SCS when the mcount-based graph tracer is enabled.

SCS and patchable-function-entry are both supported from LLVM 10.x.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 334a3d9b19df..45dfca9a98d3 100644
@@ -543,6 +543,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
 	depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
+	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
 	  shadow stack to protect function return addresses from being
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#418 lore raw

PATCH v13 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 85e4149cc5d5..409a6c1be8cc 100644
@@ -81,6 +81,10 @@ endif
 
 KBUILD_CFLAGS += $(branch-prot-flags-y)
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#419 lore raw

PATCH v13 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index 197a9ba2d5ea..ed15be0f8103 100644
@@ -58,6 +58,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#420 lore raw

PATCH v13 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#421 lore raw

PATCH v13 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#422 lore raw

PATCH v13 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index fe57f60f06a8..875b106c5d98 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#423 lore raw

PATCH v13 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 +++
 arch/arm64/include/asm/scs.h         | 46 ++++++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h | 13 ++++++++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  4 +++
 arch/arm64/kernel/entry.S            | 24 +++++++++++++--
 arch/arm64/kernel/head.S             |  6 ++++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 15 +++++++++
 9 files changed, 114 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 40fb05d96c60..c380a16533f6 100644
@@ -64,6 +64,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1025,6 +1026,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +46 -0

46 lines not on this page · raw message

arch/arm64/include/asm/thread_info.h +13 -0
index 512174a8e789..9df79c0a4c43 100644
@@ -41,6 +41,10 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*scs_base;
+	unsigned long		scs_offset;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
@@ -100,11 +104,20 @@ void arch_release_task_struct(struct task_struct *tsk);
 				 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
 				 _TIF_SYSCALL_EMU)
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+#define INIT_SCS							\
+	.scs_base	= init_shadow_call_stack,			\
+	.scs_offset	= 0,
+#else
+#define INIT_SCS
+#endif
+
 #define INIT_THREAD_INFO(tsk)						\
 {									\
 	.flags		= _TIF_FOREIGN_FPSTATE,				\
 	.preempt_count	= INIT_PREEMPT_COUNT,				\
 	.addr_limit	= KERNEL_DS,					\
+	INIT_SCS							\
 }
 
 #endif /* __ASM_THREAD_INFO_H */
arch/arm64/kernel/Makefile +1 -0
index 4e5b8ee31442..151f28521f1e 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +4 -0
index 9981a0a5a87f..d7934250b68c 100644
@@ -33,6 +33,10 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS_BASE,	offsetof(struct task_struct, thread_info.scs_base));
+  DEFINE(TSK_TI_SCS_OFFSET,	offsetof(struct task_struct, thread_info.scs_offset));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +22 -2
index ddcde093c433..244268d5ae47 100644
@@ -23,6 +23,7 @@
 #include <asm/mmu.h>
 #include <asm/processor.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/thread_info.h>
 #include <asm/asm-uaccess.h>
 #include <asm/unistd.h>
@@ -179,6 +180,8 @@ alternative_cb_end
 	apply_ssbd 1, x22, x23
 
 	ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
+
+	scs_load tsk, x20
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -343,6 +346,8 @@ alternative_else_nop_endif
 	msr	cntkctl_el1, x1
 4:
 #endif
+	scs_save tsk, x0
+
 	/* No kernel C function calls after this as user keys are set. */
 	ptrauth_keys_install_user tsk, x0, x1, x2
 
@@ -388,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -405,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	adr_this_cpu x18, irq_shadow_call_stack, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -901,6 +919,8 @@ SYM_FUNC_START(cpu_switch_to)
 	mov	sp, x9
 	msr	sp_el0, x1
 	ptrauth_keys_install_kernel x1, 1, x8, x9, x10
+	scs_save x0, x8
+	scs_load x1, x8
 	ret
 SYM_FUNC_END(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +6 -0
index 57a91032b4c2..2b01c19c5483 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -737,6 +742,7 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+	scs_load x2, x3
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/process.c +2 -0
index 56be4cbf771f..a35d3318492c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -515,6 +516,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	entry_task_switch(next);
 	uao_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +15 -0
new file mode 100644
index 000000000000..acc6741d1a40
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <asm/scs.h>
+
+/* Allocate a static per-CPU shadow stack */
+#define DEFINE_SCS(name)						\
+	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
+
+DEFINE_SCS(irq_shadow_call_stack);
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#424 lore raw

PATCH v13 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/kernel/entry.S | 14 +++++++++++++-
 arch/arm64/kernel/scs.c   |  5 +++++
 2 files changed, 18 insertions(+), 1 deletion(-)
arch/arm64/kernel/entry.S +13 -1
index 244268d5ae47..cb0516e6f963 100644
@@ -1049,13 +1049,16 @@ SYM_CODE_START(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1065,6 +1068,15 @@ SYM_CODE_START(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal, tmp=x6
+	b	4f
+3:	adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +5 -0
index acc6741d1a40..adc97f826fab 100644
@@ -13,3 +13,8 @@
 	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
 
 DEFINE_SCS(irq_shadow_call_stack);
+
+#ifdef CONFIG_ARM_SDE_INTERFACE
+DEFINE_SCS(sdei_shadow_call_stack_normal);
+DEFINE_SCS(sdei_shadow_call_stack_critical);
+#endif
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
#425 lore raw

PATCH v13 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Shadow stacks are not available in the EFI stub, filter out SCS flags.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 094eabdecfe6..b52ae8c29560 100644
@@ -32,6 +32,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+# remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.26.2.303.gf8c07b1a785-goog
maintainer
#426 lore raw
In reply to Sami Tolvanen
Quoted 34 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Quoted 11 lines

Changes in v13:
- Changed thread_info::shadow_call_stack to a base address and an offset instead, and removed the now unneeded __scs_base() and scs_save().
- Removed alignment from the kmem_cache and static allocations.
- Removed the task_set_scs() helper function.
- Moved the assembly code for loading and storing the offset in thread_info to scs_load/save macros.
- Added offset checking to scs_corrupted().
- Switched to cmpxchg_relaxed() in scs_check_usage().

OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.

The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.

Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.

maintainer
#427 lore raw
Quoted 58 lines · Ard Biesheuvel
Quoted 34 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks in memory, which means an attacker capable of reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the shadow stacks.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Quoted 11 lines

Changes in v13:
- Changed thread_info::shadow_call_stack to a base address and an offset instead, and removed the now unneeded __scs_base() and scs_save().
- Removed alignment from the kmem_cache and static allocations.
- Removed the task_set_scs() helper function.
- Moved the assembly code for loading and storing the offset in thread_info to scs_load/save macros.
- Added offset checking to scs_corrupted().
- Switched to cmpxchg_relaxed() in scs_check_usage().

OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.

The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.

Quoted 7 lines

Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.

Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(

#428 lore raw
Quoted 18 lines · Ard Biesheuvel
Quoted 10 lines

OK, so one thing that came up in an offline discussion about SCS is the way it interacts with the vmap'ed stack.

The vmap'ed stack is great for robustness, but it only works if things don't explode for other reasons in the mean time. This means the ordinary-to-shadow-call-stack size ratio should be chosen such that it is *really* unlikely you could ever overflow the shadow call stack and corrupt another task's call stack before hitting the vmap stack's guard region.

Quoted 7 lines

Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.

Quoted 2 lines

Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(

Yes, and the stack potentially overflowing into the SCS sort of defeats the purpose. I'm fine with increasing the SCS size to something safer, but using a vmapped shadow stack seems like the correct solution to this problem, at least on devices where allocating a full page isn't an issue.

Sami

#429 lore raw

From: Sami Tolvanen

Quoted 1 line

Sent: 27 April 2020 23:10

...

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

Alternatively, I wonder if there is a way we could let the SCS and ordinary stack share the [bottom of] the vmap'ed region. That would give rather nasty results if the ordinary stack overflows into the SCS, but for cases where we really recurse out of control, we could catch this occurrence on either stack, whichever one occurs first. And the nastiness -when it does occur- will not corrupt any state beyond the stack of the current task.

Quoted 7 lines
Quoted 2 lines

Hmm, I guess that would make it quite hard to keep the SCS address secret though :-(

Yes, and the stack potentially overflowing into the SCS sort of defeats the purpose. I'm fine with increasing the SCS size to something safer, but using a vmapped shadow stack seems like the correct solution to this problem, at least on devices where allocating a full page isn't an issue.

Wouldn't you do it the other way around - so shadow stack overflow corrupts the bottom of the normal stack?
That can be detected 'after the fact' in a few places (eg process switch and return to user)

Actually you might want to do syscall entry at the base of stack area, then (effectively) allocate an on-stack buffer for the shadow stack.

I'd have though that kernel code could be the shadow stack address by just reading r18?
Userspace isn't supposed to be able to get the main kernel stack address either.

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

maintainer
#430 lore raw
In reply to Sami Tolvanen

Hi Sami,

Quoted 6 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

I'm planning to queue this with the (mostly cosmetic) diff below folded in. I also have some extra patches on top which I'll send out shortly for review.

However, I really think we need to get to the bottom of the size issue since I'm highly sceptical about not being able to afford a full page for the shadow stack allocation. We can change this later so it needn't hold up the patchset, but given that Android is the only user, I'd like to make sure that if we change to use a full page upstream then that is also acceptable in AOSP.

Thanks,

Will

--->8

include/linux/compiler-clang.h +0 -2
index 18fc4d29ef27..790c0c6b8552 100644
@@ -45,6 +45,4 @@
 
 #if __has_feature(shadow_call_stack)
 # define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
-#else
-# define __noscs
 #endif
include/linux/scs.h +5 -3
index 060eeb3d1390..3f3662621a27 100644
@@ -11,7 +11,7 @@
 #include <linux/gfp.h>
 #include <linux/poison.h>
 #include <linux/sched.h>
-#include <asm/page.h>
+#include <linux/sizes.h>
 
 #ifdef CONFIG_SHADOW_CALL_STACK
 
@@ -20,7 +20,7 @@
  * architecture) provided ~40% safety margin on stack usage while keeping
  * memory allocation overhead reasonable.
  */
-#define SCS_SIZE		1024UL
+#define SCS_SIZE		SZ_1K
 #define GFP_SCS			(GFP_KERNEL | __GFP_ZERO)
 
 /* An illegal pointer value to mark the end of the shadow stack. */
@@ -29,7 +29,9 @@
 #define task_scs(tsk)		(task_thread_info(tsk)->scs_base)
 #define task_scs_offset(tsk)	(task_thread_info(tsk)->scs_offset)
 
-extern void scs_init(void);
+void scs_init(void);
+int scs_prepare(struct task_struct *tsk, int node);
+void scs_release(struct task_struct *tsk);
 
 static inline void scs_task_reset(struct task_struct *tsk)
 {
@@ -40,8 +42,6 @@ static inline void scs_task_reset(struct task_struct *tsk)
	task_scs_offset(tsk) = 0;

}

-extern int scs_prepare(struct task_struct *tsk, int node); -
static inline unsigned long *__scs_magic(void *s)

 {
	return (unsigned long *)(s + SCS_SIZE) - 1;
@@ -55,12 +55,8 @@ static inline bool scs_corrupted(struct task_struct *tsk)
		READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC);

}

-extern void scs_release(struct task_struct *tsk); -
#else /* CONFIG_SHADOW_CALL_STACK */

-#define task_scs(tsk) NULL
-
static inline void scs_init(void) {}
static inline void scs_task_reset(struct task_struct *tsk) {} static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }

kernel/scs.c +1 -2
index 2a96573f2b1b..9389c28f0853 100644
@@ -55,45 +55,37 @@ static void scs_account(struct task_struct *tsk, int account)
 
 int scs_prepare(struct task_struct *tsk, int node)
 {
-	void *s;
+	void *s = scs_alloc(node);
 
-	s = scs_alloc(node);
	if (!s)
		return -ENOMEM;
 
	task_scs(tsk) = s;
	task_scs_offset(tsk) = 0;
	scs_account(tsk, 1);
-
	return 0;

}

-#ifdef CONFIG_DEBUG_STACK_USAGE
-static unsigned long __scs_used(struct task_struct *tsk) +static void scs_check_usage(struct task_struct *tsk)
{
- unsigned long *p = task_scs(tsk);
- unsigned long *end = __scs_magic(p);
- unsigned long s = (unsigned long)p;
+ static unsigned long highest;

- while (p < end && READ_ONCE_NOCHECK(*p))
- p++;
+ unsigned long *p, prev, curr = highest, used = 0;

- return (unsigned long)p - s;
-}
+ if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
+ return;

-static void scs_check_usage(struct task_struct *tsk) -{
- static unsigned long highest;
- unsigned long used = __scs_used(tsk);
- unsigned long prev;
- unsigned long curr = highest;
+ for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
+ if (!READ_ONCE_NOCHECK(*p))
+ break;
+ used++;
+ }

	while (used > curr) {
		prev = cmpxchg_relaxed(&highest, curr, used);
 
		if (prev == curr) {
-			pr_info("%s (%d): highest shadow stack usage: "
-				"%lu bytes\n",
+			pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
				tsk->comm, task_pid_nr(tsk), used);
			break;
		}
@@ -101,21 +93,16 @@ static void scs_check_usage(struct task_struct *tsk)
		curr = prev;
	}

}
-#else
-static inline void scs_check_usage(struct task_struct *tsk) {} -#endif

void scs_release(struct task_struct *tsk)
{
- void *s;
+ void *s = task_scs(tsk);

-	s = task_scs(tsk);
	if (!s)
		return;

- WARN_ON(scs_corrupted(tsk));
+ WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n");

	scs_check_usage(tsk);
-
	scs_account(tsk, -1);
	scs_free(s);

}