Skip to contentSkip to conversation

add support for Clang's Shadow Call Stack

PATCH Superseded Sami Tolvanen 18 participants 431 messages Oct 18, 2019 – May 15, 2020 Page 2 of 3

Subsystems (16)
lore
Share & tools
lore
#167 lore raw

PATCH v4 5/17 add support for Clang's Shadow Call Stack (SCS)

In reply to Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  33 +++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 +++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/sched/sched.h           |   1 +
 kernel/scs.c                   | 169 +++++++++++++++++++++++++++++++++
 11 files changed, 296 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 79be70bf2899..e6337314f8fb 100644
@@ -846,6 +846,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +33 -0
index 5f8a5d84dbbe..5e34cbcd8d6a 100644
@@ -521,6 +521,39 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK_VMAP
+	bool
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index daad787fb795..313dbd44d576 100644
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index bcdf53125210..3fa7ba64c62d 100644
@@ -94,6 +94,7 @@
 #include <linux/livepatch.h>
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -451,6 +452,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -834,6 +837,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -893,6 +898,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index dd05a378631a..e7faeb383008 100644
@@ -6013,6 +6013,8 @@ void init_idle(struct task_struct *idle, int cpu)
 	raw_spin_lock_irqsave(&idle->pi_lock, flags);
 	raw_spin_lock(&rq->lock);
 
+	scs_task_reset(idle);
+
 	__sched_fork(0, idle);
 	idle->state = TASK_RUNNING;
 	idle->se.exec_start = sched_clock();
kernel/sched/sched.h +1 -0
index 0db2c1b3361e..c153003a011c 100644
@@ -58,6 +58,7 @@
 #include <linux/profile.h>
 #include <linux/psi.h>
 #include <linux/rcupdate_wait.h>
+#include <linux/scs.h>
 #include <linux/security.h>
 #include <linux/stop_machine.h>
 #include <linux/suspend.h>
kernel/scs.c new file +169 -0

169 lines not on this page · raw message

Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
Reviewed-by on v4 5/17
#168 lore raw

Re: [PATCH v4 05/17] add support for Clang's Shadow Call Stack (SCS)

Quoted 14 lines · Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

Quoted 2 lines

Cheers,
Miguel

reviewer
#169 lore raw

Re: [PATCH v4 05/17] add support for Clang's Shadow Call Stack (SCS)

In reply to Sami Tolvanen
Quoted 139 lines · Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  33 +++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 +++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/sched/sched.h           |   1 +
 kernel/scs.c                   | 169 +++++++++++++++++++++++++++++++++
 11 files changed, 296 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 79be70bf2899..e6337314f8fb 100644
@@ -846,6 +846,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +33 -0
index 5f8a5d84dbbe..5e34cbcd8d6a 100644
@@ -521,6 +521,39 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK_VMAP
+	bool
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +12 -0
new file mode 100644
index 000000000000..bd5ef4278b91
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#ifndef _LINUX_SCS_H
+#define _LINUX_SCS_H
+
+#include <linux/gfp.h>
+#include <linux/sched.h>

include/linux/scs.h line 22

Quoted 10 lines
include/linux/scs.h new file +10 -0
+#include <asm/page.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+/*
+ * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
+ * architecture) provided ~40% safety margin on stack usage while keeping
+ * memory allocation overhead reasonable.
+ */
+#define SCS_SIZE	1024

To make it easier to reason about type promotion rules (and avoid that we accidentaly mask out high bits when using this to generate a mask), can we please make this 1024UL?

Quoted 119 lines
include/linux/scs.h new file +35 -0
+#define GFP_SCS		(GFP_KERNEL | __GFP_ZERO)
+
+/*
+ * A random number outside the kernel's virtual address space to mark the
+ * end of the shadow stack.
+ */
+#define SCS_END_MAGIC	0xaf0194819b1635f6UL
+
+#define task_scs(tsk)	(task_thread_info(tsk)->shadow_call_stack)
+
+static inline void task_set_scs(struct task_struct *tsk, void *s)
+{
+	task_scs(tsk) = s;
+}
+
+extern void scs_init(void);
+extern void scs_task_reset(struct task_struct *tsk);
+extern int scs_prepare(struct task_struct *tsk, int node);
+extern bool scs_corrupted(struct task_struct *tsk);
+extern void scs_release(struct task_struct *tsk);
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+#define task_scs(tsk)	NULL
+
+static inline void task_set_scs(struct task_struct *tsk, void *s) {}
+static inline void scs_init(void) {}
+static inline void scs_task_reset(struct task_struct *tsk) {}
+static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
+static inline bool scs_corrupted(struct task_struct *tsk) { return false; }
+static inline void scs_release(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* _LINUX_SCS_H */
init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index daad787fb795..313dbd44d576 100644
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index bcdf53125210..3fa7ba64c62d 100644
@@ -94,6 +94,7 @@
 #include <linux/livepatch.h>
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -451,6 +452,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -834,6 +837,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -893,6 +898,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +0 -0
index dd05a378631a..e7faeb383008 100644

kernel/sched/core.c void init_idle(struct task_struct *idle, int cpu) · line 6016

Quoted 4 lines
kernel/sched/core.c +1 -0
@@ -6013,6 +6013,8 @@ void init_idle(struct task_struct *idle, int cpu)
 	raw_spin_lock_irqsave(&idle->pi_lock, flags);
 	raw_spin_lock(&rq->lock);
 
+	scs_task_reset(idle);

Could we please do this next to the kasan_unpoison_task_stack() call, Either just before, or just after?

They're boot addressing the same issue where previously live stack is being reused, and in general I'd expect them to occur at the same time (though I understand idle will be a bit different).

Quoted 9 lines
kernel/sched/core.c +1 -0
+
 	__sched_fork(0, idle);
 	idle->state = TASK_RUNNING;
 	idle->se.exec_start = sched_clock();
kernel/sched/sched.h +0 -0
index 0db2c1b3361e..c153003a011c 100644

kernel/sched/sched.h line 64

Quoted 7 lines
kernel/sched/sched.h +1 -0
@@ -58,6 +58,7 @@
 #include <linux/profile.h>
 #include <linux/psi.h>
 #include <linux/rcupdate_wait.h>
+#include <linux/scs.h>
 #include <linux/security.h>
 #include <linux/stop_machine.h>
 #include <linux/suspend.h>

This include looks extraneous.

Quoted 26 lines
kernel/scs.c new file +20 -0
new file mode 100644
index 000000000000..7c1a40020754
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/cpuhotplug.h>
+#include <linux/mm.h>
+#include <linux/mmzone.h>
+#include <linux/scs.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <asm/scs.h>
+
+static inline void *__scs_base(struct task_struct *tsk)
+{
+	/*
+	 * We allow architectures to use the shadow_call_stack field in
+	 * struct thread_info to store the current shadow stack pointer

kernel/scs.c line 30

Quoted 10 lines
kernel/scs.c new file +10 -0
+	 * during context switches.
+	 *
+	 * This allows the implementation to also clear the field when
+	 * the task is active to avoid keeping pointers to the current
+	 * task's shadow stack in memory. This can make it harder for an
+	 * attacker to locate the shadow stack, but also requires us to
+	 * compute the base address when needed.
+	 *
+	 * We assume the stack is aligned to SCS_SIZE.
+	 */

How about:

/*
* To minimize risk the of exposure, architectures may clear a
* task's thread_info::shadow_call_stack while that task is
* running, and only save/restore the active shadow call stack
* pointer when the usual register may be clobbered (e.g. across
* context switches).
*
* The shadow call stack is aligned to SCS_SIZE, and grows
* upwards, so we can mask out the low bits to extract the base
* when the task is not running.
*/

... which I think makes the lifetime and constraints a bit clearer.

kernel/scs.c line 31

Quoted 1 line
kernel/scs.c new file +1 -0
+	return (void *)((uintptr_t)task_scs(tsk) & ~(SCS_SIZE - 1));

We usually use unsigned long ratehr than uintptr_t. Could we please use that for consistency?

The kernel relies on sizeof(unsigned long) == sizeof(void *) tree-wide, so that doesn't cause issues for us here.

Similarly, as suggested above, it would be easier to reason about this knowing that SCS_SIZE is an unsigned long. While IIUC we'd get sign extension here when it's promoted, giving the definition a UL suffix minimizes the scope for error.

kernel/scs.c line 37

Quoted 6 lines
kernel/scs.c new file +6 -0
+}
+
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+
+/* Keep a cache of shadow stacks */
+#define SCS_CACHE_SIZE 2

How about:

/* Matches NR_CACHED_STACKS for VMAP_STACK */
#define NR_CACHED_SCS 2

... which explains where the number came from, and avoids confusion that the SIZE is a byte size rather than number of elements.

Quoted 25 lines
kernel/scs.c new file +25 -0
+static DEFINE_PER_CPU(void *, scs_cache[SCS_CACHE_SIZE]);
+
+static void *scs_alloc(int node)
+{
+	int i;
+
+	for (i = 0; i < SCS_CACHE_SIZE; i++) {
+		void *s;
+
+		s = this_cpu_xchg(scs_cache[i], NULL);
+		if (s) {
+			memset(s, 0, SCS_SIZE);
+			return s;
+		}
+	}
+
+	/*
+	 * We allocate a full page for the shadow stack, which should be
+	 * more than we need. Check the assumption nevertheless.
+	 */
+	BUILD_BUG_ON(SCS_SIZE > PAGE_SIZE);
+
+	return __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+				    VMALLOC_START, VMALLOC_END,
+				    GFP_SCS, PAGE_KERNEL, 0,

kernel/scs.c line 72

Quoted 10 lines
kernel/scs.c new file +10 -0
+				    node, __builtin_return_address(0));
+}
+
+static void scs_free(void *s)
+{
+	int i;
+
+	for (i = 0; i < SCS_CACHE_SIZE; i++)
+		if (this_cpu_cmpxchg(scs_cache[i], 0, s) == 0)
+			return;

Here we should compare to NULL rather than 0.

Quoted 11 lines
kernel/scs.c new file +11 -0
+
+	vfree_atomic(s);
+}
+
+static int scs_cleanup(unsigned int cpu)
+{
+	int i;
+	void **cache = per_cpu_ptr(scs_cache, cpu);
+
+	for (i = 0; i < SCS_CACHE_SIZE; i++) {
+		vfree(cache[i]);

kernel/scs.c line 93

Quoted 10 lines
kernel/scs.c new file +10 -0
+		cache[i] = NULL;
+	}
+
+	return 0;
+}
+
+void __init scs_init(void)
+{
+	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
+		scs_cleanup);

We probably want to do something if this call fails. It looks like we'd only leak two pages (and we'd be able to use them if/when that CPU is brought back online. A WARN_ON() is probably fine.

Thanks,
Mark.

Quoted 79 lines
kernel/scs.c new file +76 -0
+}
+
+#else /* !CONFIG_SHADOW_CALL_STACK_VMAP */
+
+static struct kmem_cache *scs_cache;
+
+static inline void *scs_alloc(int node)
+{
+	return kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
+}
+
+static inline void scs_free(void *s)
+{
+	kmem_cache_free(scs_cache, s);
+}
+
+void __init scs_init(void)
+{
+	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
+				0, NULL);
+	WARN_ON(!scs_cache);
+}
+
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+
+static inline unsigned long *scs_magic(struct task_struct *tsk)
+{
+	return (unsigned long *)(__scs_base(tsk) + SCS_SIZE) - 1;
+}
+
+static inline void scs_set_magic(struct task_struct *tsk)
+{
+	*scs_magic(tsk) = SCS_END_MAGIC;
+}
+
+void scs_task_reset(struct task_struct *tsk)
+{
+	/*
+	 * Reset the shadow stack to the base address in case the task
+	 * is reused.
+	 */
+	task_set_scs(tsk, __scs_base(tsk));
+}
+
+int scs_prepare(struct task_struct *tsk, int node)
+{
+	void *s;
+
+	s = scs_alloc(node);
+	if (!s)
+		return -ENOMEM;
+
+	task_set_scs(tsk, s);
+	scs_set_magic(tsk);
+
+	return 0;
+}
+
+bool scs_corrupted(struct task_struct *tsk)
+{
+	return *scs_magic(tsk) != SCS_END_MAGIC;
+}
+
+void scs_release(struct task_struct *tsk)
+{
+	void *s;
+
+	s = __scs_base(tsk);
+	if (!s)
+		return;
+
+	WARN_ON(scs_corrupted(tsk));
+
+	task_set_scs(tsk, NULL);
+	scs_free(s);
+}
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#170 lore raw

Re: [PATCH v4 05/17] add support for Clang's Shadow Call Stack (SCS)

Quoted 10 lines · Mark Rutland

include/linux/scs.h line 22

Quoted 6 lines
include/linux/scs.h new file +6 -0
+/*
+ * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
+ * architecture) provided ~40% safety margin on stack usage while keeping
+ * memory allocation overhead reasonable.
+ */
+#define SCS_SIZE     1024

To make it easier to reason about type promotion rules (and avoid that we accidentaly mask out high bits when using this to generate a mask), can we please make this 1024UL?

Sure.

Quoted 6 lines
Quoted 6 lines
kernel/sched/core.c +0 -0
@@ -6013,6 +6013,8 @@ void init_idle(struct task_struct *idle, int cpu)
      raw_spin_lock_irqsave(&idle->pi_lock, flags);
      raw_spin_lock(&rq->lock);
 
Quoted 8 lines

kernel/sched/core.c void init_idle(struct task_struct *idle, int cpu) · line 6016

Quoted 1 line
kernel/sched/core.c +1 -0
+     scs_task_reset(idle);

Could we please do this next to the kasan_unpoison_task_stack() call, Either just before, or just after?

They're boot addressing the same issue where previously live stack is being reused, and in general I'd expect them to occur at the same time (though I understand idle will be a bit different).

Good point, I'll move this.

Quoted 4 lines
Quoted 4 lines
kernel/sched/sched.h +0 -0
@@ -58,6 +58,7 @@
 #include <linux/profile.h>
Quoted 8 lines

kernel/sched/sched.h line 64

Quoted 6 lines
kernel/sched/sched.h +1 -0
 #include <linux/psi.h>
 #include <linux/rcupdate_wait.h>
+#include <linux/scs.h>
 #include <linux/security.h>
 #include <linux/stop_machine.h>
 #include <linux/suspend.h>

This include looks extraneous.

I added this to sched.h, because most of the includes used in kernel/sched appear to be there, but I can move this to kernel/sched/core.c instead.

Quoted 30 lines
Quoted 5 lines
kernel/sched/sched.h +5 -0
+static inline void *__scs_base(struct task_struct *tsk)
+{
+     /*
+      * We allow architectures to use the shadow_call_stack field in
+      * struct thread_info to store the current shadow stack pointer

kernel/sched/sched.h

Quoted 10 lines
kernel/sched/sched.h +10 -0
+      * during context switches.
+      *
+      * This allows the implementation to also clear the field when
+      * the task is active to avoid keeping pointers to the current
+      * task's shadow stack in memory. This can make it harder for an
+      * attacker to locate the shadow stack, but also requires us to
+      * compute the base address when needed.
+      *
+      * We assume the stack is aligned to SCS_SIZE.
+      */

How about:

/*
* To minimize risk the of exposure, architectures may clear a
* task's thread_info::shadow_call_stack while that task is
* running, and only save/restore the active shadow call stack
* pointer when the usual register may be clobbered (e.g. across
* context switches).
*
* The shadow call stack is aligned to SCS_SIZE, and grows
* upwards, so we can mask out the low bits to extract the base
* when the task is not running.
*/

Quoted 1 line

... which I think makes the lifetime and constraints a bit clearer.

Sounds good to me, thanks.

Quoted 5 lines
Quoted 1 line

+ return (void *)((uintptr_t)task_scs(tsk) & ~(SCS_SIZE - 1));

We usually use unsigned long ratehr than uintptr_t. Could we please use that for consistency?

Quoted 7 lines

The kernel relies on sizeof(unsigned long) == sizeof(void *) tree-wide, so that doesn't cause issues for us here.

Similarly, as suggested above, it would be easier to reason about this knowing that SCS_SIZE is an unsigned long. While IIUC we'd get sign extension here when it's promoted, giving the definition a UL suffix minimizes the scope for error.

OK, I'll switch to unsigned long.

Quoted 10 lines

kernel/sched/sched.h

Quoted 2 lines
kernel/sched/sched.h +2 -0
+/* Keep a cache of shadow stacks */
+#define SCS_CACHE_SIZE 2

How about:

/* Matches NR_CACHED_STACKS for VMAP_STACK */
#define NR_CACHED_SCS 2

... which explains where the number came from, and avoids confusion that the SIZE is a byte size rather than number of elements.

Agreed, that sounds better.

Quoted 9 lines

kernel/sched/sched.h

Quoted 7 lines
kernel/sched/sched.h +7 -0
+static void scs_free(void *s)
+{
+     int i;
+
+     for (i = 0; i < SCS_CACHE_SIZE; i++)
+             if (this_cpu_cmpxchg(scs_cache[i], 0, s) == 0)
+                     return;

Here we should compare to NULL rather than 0.

Ack.

Quoted 8 lines

kernel/sched/sched.h

Quoted 4 lines
kernel/sched/sched.h +4 -0
+void __init scs_init(void)
+{
+     cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
+             scs_cleanup);

We probably want to do something if this call fails. It looks like we'd only leak two pages (and we'd be able to use them if/when that CPU is brought back online. A WARN_ON() is probably fine.

fork_init() in kernel/fork.c lets this fail quietly, but adding a WARN_ON seems fine.

I will include these changes in v5.

Sami

#171 lore raw

PATCH v4 6/17 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 19 +++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 41 insertions(+)
drivers/base/node.c +6 -0
index 296546ffed6c..111e58ec231e 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index bda20282746b..fcb8c1708f9e 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +19 -0
index 7c1a40020754..7780fc4e29ac 100644
@@ -11,6 +11,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -74,6 +75,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -107,6 +113,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -135,6 +146,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -145,6 +162,7 @@ int scs_prepare(struct task_struct *tsk, int node)
 
 	task_set_scs(tsk, s);
 	scs_set_magic(tsk);
+	scs_account(tsk, 1);
 
 	return 0;
 }
@@ -164,6 +182,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index ecc3dbad606b..fe17d69d98a7 100644
@@ -5361,6 +5361,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5382,6 +5385,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 6afc892a148a..9fe4afe670fe 100644
@@ -1118,6 +1118,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
maintainer
#172 lore raw

Re: [PATCH v4 06/17] scs: add accounting

Quoted 61 lines · Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 19 +++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 41 insertions(+)
drivers/base/node.c +3 -0
index 296546ffed6c..111e58ec231e 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device 
 *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
drivers/base/node.c +3 -0
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device 
 *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 
 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
drivers/base/node.c +0 -0
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m,
 void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
include/linux/mmzone.h +0 -0
index bda20282746b..fcb8c1708f9e 100644

include/linux/mmzone.h enum zone_stat_item { · line 203

Quoted 4 lines
include/linux/mmzone.h +1 -0
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)

Is there any reason why you're not consistently using only one of "#if IS_ENABLED(...)" or "#ifdef ...", but instead a mix of both?

Thanks,

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#173 lore raw

Re: [PATCH v4 06/17] scs: add accounting

Quoted 2 lines · Marc Zyngier

Is there any reason why you're not consistently using only one of "#if IS_ENABLED(...)" or "#ifdef ...", but instead a mix of both?

This is to match the style already used in each file. For example, fs/proc/meminfo.c uses #ifdef for other configs in the same function, and include/linux/mmzone.h uses #if IS_ENABLED(...).

Sami

maintainer
#174 lore raw

Re: [PATCH v4 06/17] scs: add accounting

Quoted 7 lines · Sami Tolvanen
Quoted 2 lines · Marc Zyngier

Is there any reason why you're not consistently using only one of "#if IS_ENABLED(...)" or "#ifdef ...", but instead a mix of both?

This is to match the style already used in each file. For example, fs/proc/meminfo.c uses #ifdef for other configs in the same function, and include/linux/mmzone.h uses #if IS_ENABLED(...).

Ah, fair enough.

        M.
Signature · 6 lines
Jazz is not dead. It just smells funny...

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
#175 lore raw

PATCH v4 7/17 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 7780fc4e29ac..67c43af627d1 100644
@@ -167,6 +167,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(tsk);
+	uintptr_t s = (uintptr_t)p;
+
+	while (p < end && *p)
+		p++;
+
+	return (uintptr_t)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s: highest shadow stack usage %lu bytes\n",
+			__func__, used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	return *scs_magic(tsk) != SCS_END_MAGIC;
@@ -181,6 +219,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
Reviewed-by on v4 7/17
#176 lore raw

Re: [PATCH v4 07/17] scs: add support for stack usage debugging

Quoted 4 lines · Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

Thanks for helping me find this Kconfig. :) :)

-Kees

Quoted 64 lines
---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 7780fc4e29ac..67c43af627d1 100644
@@ -167,6 +167,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(tsk);
+	uintptr_t s = (uintptr_t)p;
+
+	while (p < end && *p)
+		p++;
+
+	return (uintptr_t)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s: highest shadow stack usage %lu bytes\n",
+			__func__, used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	return *scs_magic(tsk) != SCS_END_MAGIC;
@@ -181,6 +219,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
Signature · 1 line
Kees Cook
reviewer
#177 lore raw

Re: [PATCH v4 07/17] scs: add support for stack usage debugging

In reply to Sami Tolvanen
Quoted 13 lines · Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +0 -0
index 7780fc4e29ac..67c43af627d1 100644

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 175

Quoted 9 lines
kernel/scs.c +6 -0
@@ -167,6 +167,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(tsk);
+	uintptr_t s = (uintptr_t)p;

As previously, please use unsigned long for consistency.

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 178

Quoted 3 lines
kernel/scs.c +3 -0
+
+	while (p < end && *p)
+		p++;

I think this is the only place where we legtimately access the shadow call stack directly. When using SCS and KASAN, are the compiler-generated accesses to the SCS instrumented?

If not, it might make sense to make this:

	while (p < end && READ_ONCE_NOCKECK(*p))

... and poison the allocation from KASAN's PoV, so that we can find unintentional accesses more easily.

Mark.

Quoted 43 lines
kernel/scs.c +30 -0
+
+	return (uintptr_t)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s: highest shadow stack usage %lu bytes\n",
+			__func__, used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	return *scs_magic(tsk) != SCS_END_MAGIC;
@@ -181,6 +219,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#178 lore raw

Re: [PATCH v4 07/17] scs: add support for stack usage debugging

Quoted 8 lines · Mark Rutland

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 175

Quoted 6 lines
kernel/scs.c +6 -0
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+     unsigned long *p = __scs_base(tsk);
+     unsigned long *end = scs_magic(tsk);
+     uintptr_t s = (uintptr_t)p;

As previously, please use unsigned long for consistency.

Ack.

Quoted 5 lines

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 178

Quoted 2 lines
kernel/scs.c +2 -0
+     while (p < end && *p)
+             p++;

I think this is the only place where we legtimately access the shadow call stack directly.

There's also scs_corrupted, which checks that the end magic is intact.

Quoted 9 lines

When using SCS and KASAN, are the compiler-generated accesses to the SCS instrumented?

If not, it might make sense to make this:

        while (p < end && READ_ONCE_NOCKECK(*p))

... and poison the allocation from KASAN's PoV, so that we can find unintentional accesses more easily.

Sure, that makes sense. I can poison the allocation for the non-vmalloc case, I'll just need to refactor scs_set_magic to happen before the poisoning.

Sami

reviewer
#179 lore raw

Re: [PATCH v4 07/17] scs: add support for stack usage debugging

Quoted 12 lines · Sami Tolvanen
Quoted 8 lines · Mark Rutland

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 175

Quoted 6 lines
kernel/scs.c +6 -0
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+     unsigned long *p = __scs_base(tsk);
+     unsigned long *end = scs_magic(tsk);
+     uintptr_t s = (uintptr_t)p;

As previously, please use unsigned long for consistency.

Ack.

Quoted 7 lines
Quoted 5 lines

kernel/scs.c int scs_prepare(struct task_struct *tsk, int node) · line 178

Quoted 2 lines
kernel/scs.c +2 -0
+     while (p < end && *p)
+             p++;

I think this is the only place where we legtimately access the shadow call stack directly.

There's also scs_corrupted, which checks that the end magic is intact.

Ah, true. I missed that.

Quoted 5 lines
Quoted 5 lines

When using SCS and KASAN, are the compiler-generated accesses to the SCS instrumented?

If not, it might make sense to make this:

Quoted 8 lines
Quoted 4 lines
        while (p < end && READ_ONCE_NOCKECK(*p))

... and poison the allocation from KASAN's PoV, so that we can find unintentional accesses more easily.

Sure, that makes sense. I can poison the allocation for the non-vmalloc case, I'll just need to refactor scs_set_magic to happen before the poisoning.

Sounds good!

Mark.

#180 lore raw

PATCH v4 8/17 kprobes: fix compilation without CONFIG_KRETPROBES

In reply to Sami Tolvanen

kprobe_on_func_entry and arch_kprobe_on_func_entry need to be available even if CONFIG_KRETPROBES is not selected.

---
 kernel/kprobes.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
kernel/kprobes.c +19 -19
index 53534aa258a6..b5e20a4669b8 100644
@@ -1829,6 +1829,25 @@ unsigned long __weak arch_deref_entry_point(void *entry)
 	return (unsigned long)entry;
 }
 
+bool __weak arch_kprobe_on_func_entry(unsigned long offset)
+{
+	return !offset;
+}
+
+bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
+{
+	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
+
+	if (IS_ERR(kp_addr))
+		return false;
+
+	if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
+						!arch_kprobe_on_func_entry(offset))
+		return false;
+
+	return true;
+}
+
 #ifdef CONFIG_KRETPROBES
 /*
  * This kprobe pre_handler is registered with every kretprobe. When probe
@@ -1885,25 +1904,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 }
 NOKPROBE_SYMBOL(pre_handler_kretprobe);
 
-bool __weak arch_kprobe_on_func_entry(unsigned long offset)
-{
-	return !offset;
-}
-
-bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
-{
-	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
-
-	if (IS_ERR(kp_addr))
-		return false;
-
-	if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
-						!arch_kprobe_on_func_entry(offset))
-		return false;
-
-	return true;
-}
-
 int register_kretprobe(struct kretprobe *rp)
 {
 	int ret = 0;
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
maintainer Acked-by on v4 8/17
#181 lore raw

Re: [PATCH v4 08/17] kprobes: fix compilation without CONFIG_KRETPROBES

Quoted 6 lines · On Fri, 1 Nov 2019 15:11:41 -0700 Sami Tolvanen <[email protected]>…

kprobe_on_func_entry and arch_kprobe_on_func_entry need to be available even if CONFIG_KRETPROBES is not selected.

-- Steve

Quoted 60 lines
---
 kernel/kprobes.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
kernel/kprobes.c +19 -19
index 53534aa258a6..b5e20a4669b8 100644
@@ -1829,6 +1829,25 @@ unsigned long __weak arch_deref_entry_point(void *entry)
 	return (unsigned long)entry;
 }
 
+bool __weak arch_kprobe_on_func_entry(unsigned long offset)
+{
+	return !offset;
+}
+
+bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
+{
+	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
+
+	if (IS_ERR(kp_addr))
+		return false;
+
+	if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
+						!arch_kprobe_on_func_entry(offset))
+		return false;
+
+	return true;
+}
+
 #ifdef CONFIG_KRETPROBES
 /*
  * This kprobe pre_handler is registered with every kretprobe. When probe
@@ -1885,25 +1904,6 @@ static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 }
 NOKPROBE_SYMBOL(pre_handler_kretprobe);
 
-bool __weak arch_kprobe_on_func_entry(unsigned long offset)
-{
-	return !offset;
-}
-
-bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
-{
-	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
-
-	if (IS_ERR(kp_addr))
-		return false;
-
-	if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
-						!arch_kprobe_on_func_entry(offset))
-		return false;
-
-	return true;
-}
-
 int register_kretprobe(struct kretprobe *rp)
 {
 	int ret = 0;
#182 lore raw

PATCH v4 9/17 arm64: kprobes: fix kprobes without CONFIG_KRETPROBES

In reply to Sami Tolvanen

This allows CONFIG_KRETPROBES to be disabled without disabling kprobes entirely.

---
 arch/arm64/kernel/probes/kprobes.c | 2 ++
 1 file changed, 2 insertions(+)
arch/arm64/kernel/probes/kprobes.c +2 -0
index c4452827419b..98230ae979ca 100644
@@ -551,6 +551,7 @@ void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
 	return (void *)orig_ret_address;
 }
 
+#ifdef CONFIG_KRETPROBES
 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
 				      struct pt_regs *regs)
 {
@@ -564,6 +565,7 @@ int __kprobes arch_trampoline_kprobe(struct kprobe *p)
 {
 	return 0;
 }
+#endif
 
 int __init arch_init_kprobes(void)
 {
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#183 lore raw

PATCH v4 10/17 arm64: disable kretprobes with SCS

In reply to Sami Tolvanen

With CONFIG_KRETPROBES, function return addresses are modified to redirect control flow to kretprobe_trampoline. This is incompatible with SCS.

---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index 3f047afb982c..e7b57a8a5531 100644
@@ -165,7 +165,7 @@ config ARM64
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_KPROBES
-	select HAVE_KRETPROBES
+	select HAVE_KRETPROBES if !SHADOW_CALL_STACK
 	select HAVE_GENERIC_VDSO
 	select IOMMU_DMA if IOMMU_SUPPORT
 	select IRQ_DOMAIN
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer
#184 lore raw

Re: [PATCH v4 10/17] arm64: disable kretprobes with SCS

Quoted 3 lines · Sami Tolvanen

With CONFIG_KRETPROBES, function return addresses are modified to redirect control flow to kretprobe_trampoline. This is incompatible with SCS.

I'm a bit confused as to why that's the case -- could you please elaborate on how this is incompatible?

IIUC kretrobes works by patching the function entry point with a BRK, so that it can modify the LR _before_ it is saved to the stack. I don't see how SCS affects that.

When the instrumented function returns, it'll balance its SCS state, then "return" to kretprobe_trampoline. Since kretprobe_trampoline is plain assembly, it doesn't have SCS, and can modify the LR live, as it does.

So functionally, that appears to work. What am I missing?

Thanks,
Mark.

Quoted 22 lines
---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index 3f047afb982c..e7b57a8a5531 100644
@@ -165,7 +165,7 @@ config ARM64
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_KPROBES
-	select HAVE_KRETPROBES
+	select HAVE_KRETPROBES if !SHADOW_CALL_STACK
 	select HAVE_GENERIC_VDSO
 	select IOMMU_DMA if IOMMU_SUPPORT
 	select IRQ_DOMAIN
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#185 lore raw

Re: [PATCH v4 10/17] arm64: disable kretprobes with SCS

Quoted 6 lines · Mark Rutland

I'm a bit confused as to why that's the case -- could you please elaborate on how this is incompatible?

IIUC kretrobes works by patching the function entry point with a BRK, so that it can modify the LR _before_ it is saved to the stack. I don't see how SCS affects that.

You're correct. While this may not be optimal for reducing attack surface, I just tested this to confirm that there's no functional conflict. I'll drop this and related patches from v5.

Sami

reviewer
#186 lore raw

Re: [PATCH v4 10/17] arm64: disable kretprobes with SCS

Quoted 11 lines · Sami Tolvanen
Quoted 6 lines · Mark Rutland

I'm a bit confused as to why that's the case -- could you please elaborate on how this is incompatible?

IIUC kretrobes works by patching the function entry point with a BRK, so that it can modify the LR _before_ it is saved to the stack. I don't see how SCS affects that.

You're correct. While this may not be optimal for reducing attack surface, I just tested this to confirm that there's no functional conflict. I'll drop this and related patches from v5.

Great; thanks for confirming!

Mark.

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#187 lore raw

PATCH v4 11/17 arm64: disable function graph tracing with SCS

In reply to Sami Tolvanen

With CONFIG_FUNCTION_GRAPH_TRACER, function return addresses are modified in ftrace_graph_caller and prepare_ftrace_return to redirect control flow to ftrace_return_to_handler. This is incompatible with SCS.

---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index e7b57a8a5531..42867174920f 100644
@@ -148,7 +148,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer
#188 lore raw

Re: [PATCH v4 11/17] arm64: disable function graph tracing with SCS

Quoted 4 lines · Sami Tolvanen

With CONFIG_FUNCTION_GRAPH_TRACER, function return addresses are modified in ftrace_graph_caller and prepare_ftrace_return to redirect control flow to ftrace_return_to_handler. This is incompatible with SCS.

Can you please elaborate on _how_ this is incompatible in the commit message?

For example, it's not clear to me if you mean that's functionally incompatible, or if you're trying to remove return-altering gadgets.

If there's a functional incompatibility, please spell that out a bit more clearly. Likewise if this is about minimizing the set of places that can mess with control-flow outside of usual function conventions.

Thanks,
Mark.

Quoted 23 lines
---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index e7b57a8a5531..42867174920f 100644
@@ -148,7 +148,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#189 lore raw

Re: [PATCH v4 11/17] arm64: disable function graph tracing with SCS

Quoted 9 lines · Mark Rutland

Can you please elaborate on _how_ this is incompatible in the commit message?

For example, it's not clear to me if you mean that's functionally incompatible, or if you're trying to remove return-altering gadgets.

If there's a functional incompatibility, please spell that out a bit more clearly. Likewise if this is about minimizing the set of places that can mess with control-flow outside of usual function conventions.

Sure, I'll add a better description in v5. In this case, the return address is modified in the kernel stack, which means the changes are ignored with SCS.

Sami

reviewer
#190 lore raw

Re: [PATCH v4 11/17] arm64: disable function graph tracing with SCS

Quoted 7 lines · Sami Tolvanen
Quoted 6 lines · Mark Rutland

Can you please elaborate on _how_ this is incompatible in the commit message?

For example, it's not clear to me if you mean that's functionally incompatible, or if you're trying to remove return-altering gadgets.

Quoted 7 lines
Quoted 3 lines

If there's a functional incompatibility, please spell that out a bit more clearly. Likewise if this is about minimizing the set of places that can mess with control-flow outside of usual function conventions.

Sure, I'll add a better description in v5. In this case, the return address is modified in the kernel stack, which means the changes are ignored with SCS.

Ok, that makes sense to me. I'd suggest something like:

| The graph tracer hooks returns by modifying frame records on the
| (regular) stack, but with SCS the return address is taken from the
| shadow stack, and the value in the frame record has no effect. As we
| don't currently have a mechanism to determine the corresponding slot
| on the shadow stack (and to pass this through the ftrace
| infrastructure), for now let's disable the graph tracer when SCS is
| enabled.

... as I suspect with some rework of the trampoline and common ftrace code we'd be able to correctly manipulate the shadow stack for this. Similarly, if clang gained -fpatchable-funciton-etnry, we'd get that for free.

Thanks,
Mark.

#192 lore raw

Re: [PATCH v4 11/17] arm64: disable function graph tracing with SCS

In reply to Mark Rutland
Quoted 15 lines · Mark Rutland
Quoted 3 lines · Sami Tolvanen

Sure, I'll add a better description in v5. In this case, the return address is modified in the kernel stack, which means the changes are ignored with SCS.

Ok, that makes sense to me. I'd suggest something like:

| The graph tracer hooks returns by modifying frame records on the
| (regular) stack, but with SCS the return address is taken from the
| shadow stack, and the value in the frame record has no effect. As we
| don't currently have a mechanism to determine the corresponding slot
| on the shadow stack (and to pass this through the ftrace
| infrastructure), for now let's disable the graph tracer when SCS is
| enabled.
Quoted 4 lines

... as I suspect with some rework of the trampoline and common ftrace code we'd be able to correctly manipulate the shadow stack for this. Similarly, if clang gained -fpatchable-funciton-etnry, we'd get that for free.

That sounds good to me. Thanks, Mark.

Sami

#193 lore raw

PATCH v4 12/17 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 2c0238ce0551..ef76101201b2 100644
@@ -72,6 +72,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#194 lore raw

PATCH v4 13/17 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +10 -0
index fdabf40a83c8..5616dc52a033 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -73,6 +75,9 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #96]
+#endif
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +94,11 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x0, #96]
+	/* Clear the SCS pointer from the state buffer */
+	str	xzr, [x0, #96]
+#endif
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
maintainer
#195 lore raw

Re: [PATCH v4 13/17] arm64: preserve x18 when CPU is suspended

Quoted 39 lines · Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +2 -0
index fdabf40a83c8..5616dc52a033 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in 

<asm/suspend.h>.
*/
ENTRY(cpu_do_suspend)

arch/arm64/mm/proc.S +0 -0
 	mrs	x2, tpidr_el0

arch/arm64/mm/proc.S alternative_endif · line 80

Quoted 6 lines
arch/arm64/mm/proc.S +3 -0
@@ -73,6 +75,9 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #96]
+#endif

Do we need the #ifdefery here? We didn't add that to the KVM path, and I'd feel better having a single behaviour, specially when NR_CTX_REGS is unconditionally sized to hold 13 regs.

Quoted 5 lines
arch/arm64/mm/proc.S +0 -0
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +94,11 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]

arch/arm64/mm/proc.S ENTRY(cpu_do_resume) · line 104

Quoted 10 lines
arch/arm64/mm/proc.S +5 -0
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x0, #96]
+	/* Clear the SCS pointer from the state buffer */
+	str	xzr, [x0, #96]
+#endif
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4

Thanks,

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#196 lore raw

Re: [PATCH v4 13/17] arm64: preserve x18 when CPU is suspended

Quoted 5 lines · Marc Zyngier
Quoted 5 lines

ENTRY(cpu_do_suspend)
mrs x2, tpidr_el0

arch/arm64/mm/proc.S +0 -0
@@ -73,6 +75,9 @@ alternative_endif
      stp     x8, x9, [x0, #48]
      stp     x10, x11, [x0, #64]
Quoted 8 lines

arch/arm64/mm/proc.S alternative_endif · line 80

Quoted 4 lines
arch/arm64/mm/proc.S +3 -0
      stp     x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+     str     x18, [x0, #96]
+#endif

Do we need the #ifdefery here? We didn't add that to the KVM path, and I'd feel better having a single behaviour, specially when NR_CTX_REGS is unconditionally sized to hold 13 regs.

I'm fine with dropping the ifdefs here in v5 unless someone objects to this.

Sami

#197 lore raw

Re: [PATCH v4 13/17] arm64: preserve x18 when CPU is suspended

Quoted 16 lines · Sami Tolvanen
Quoted 5 lines · Marc Zyngier
Quoted 5 lines

ENTRY(cpu_do_suspend)
mrs x2, tpidr_el0

arch/arm64/mm/proc.S +0 -0
@@ -73,6 +75,9 @@ alternative_endif
      stp     x8, x9, [x0, #48]
      stp     x10, x11, [x0, #64]
Quoted 8 lines

arch/arm64/mm/proc.S alternative_endif · line 80

Quoted 4 lines
arch/arm64/mm/proc.S +3 -0
      stp     x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+     str     x18, [x0, #96]
+#endif

Do we need the #ifdefery here? We didn't add that to the KVM path, and I'd feel better having a single behaviour, specially when NR_CTX_REGS is unconditionally sized to hold 13 regs.

Quoted 1 line

I'm fine with dropping the ifdefs here in v5 unless someone objects to this.

Oh, yeah I guess it would be good to be consistent. Rather than drop the ifdefs, would you (Marc) be ok with conditionally setting NR_CTX_REGS based on CONFIG_SHADOW_CALL_STACK, and doing so in KVM? (So 3 ifdefs, rather than 0)?

Without any conditionals or comments, it's not clear why x18 is being saved and restored (unless git blame survives, or a comment is added in place of the ifdefs in v6).

Signature · 7 lines
Thanks,
~Nick Desaulniers

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
#198 lore raw

Re: [PATCH v4 13/17] arm64: preserve x18 when CPU is suspended

Quoted 20 lines · Nick Desaulniers
Quoted 16 lines · Sami Tolvanen
Quoted 5 lines · Marc Zyngier
Quoted 5 lines

ENTRY(cpu_do_suspend)
mrs x2, tpidr_el0

arch/arm64/mm/proc.S +0 -0
@@ -73,6 +75,9 @@ alternative_endif
      stp     x8, x9, [x0, #48]
      stp     x10, x11, [x0, #64]
Quoted 8 lines

arch/arm64/mm/proc.S alternative_endif · line 80

Quoted 4 lines
arch/arm64/mm/proc.S +3 -0
      stp     x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+     str     x18, [x0, #96]
+#endif

Do we need the #ifdefery here? We didn't add that to the KVM path, and I'd feel better having a single behaviour, specially when NR_CTX_REGS is unconditionally sized to hold 13 regs.

Quoted 1 line

I'm fine with dropping the ifdefs here in v5 unless someone objects to this.

Quoted 8 lines

Oh, yeah I guess it would be good to be consistent. Rather than drop the ifdefs, would you (Marc) be ok with conditionally setting NR_CTX_REGS based on CONFIG_SHADOW_CALL_STACK, and doing so in KVM? (So 3 ifdefs, rather than 0)?

Without any conditionals or comments, it's not clear why x18 is being saved and restored (unless git blame survives, or a comment is added in place of the ifdefs in v6).

True. Clearing the sleep state buffer in cpu_do_resume is also pointless without CONFIG_SHADOW_CALL_STACK, so if the ifdefs are removed, some kind of an explanation is needed there.

Sami

maintainer
#199 lore raw

Re: [PATCH v4 13/17] arm64: preserve x18 when CPU is suspended

Quoted 30 lines · Sami Tolvanen
Quoted 28 lines · Nick Desaulniers
Quoted 2 lines · Sami Tolvanen

On Mon, Nov 4, 2019 at 5:20 AM Marc Zyngier <[email protected]>

Quoted 11 lines · wrote:
Quoted 11 lines

arch/arm64/mm/proc.S alternative_endif · line 80

Quoted 9 lines

ENTRY(cpu_do_suspend)
mrs x2, tpidr_el0

arch/arm64/mm/proc.S +3 -0
@@ -73,6 +75,9 @@ alternative_endif
      stp     x8, x9, [x0, #48]
      stp     x10, x11, [x0, #64]
      stp     x12, x13, [x0, #80]
+#ifdef CONFIG_SHADOW_CALL_STACK
+     str     x18, [x0, #96]
+#endif

Do we need the #ifdefery here? We didn't add that to the KVM

path,

Quoted 4 lines
Quoted 2 lines

and I'd feel better having a single behaviour, specially when NR_CTX_REGS is unconditionally sized to hold 13 regs.

I'm fine with dropping the ifdefs here in v5 unless someone

objects to this.

Oh, yeah I guess it would be good to be consistent. Rather than drop the ifdefs, would you (Marc) be ok with conditionally setting NR_CTX_REGS based on CONFIG_SHADOW_CALL_STACK, and doing so in KVM? (So 3 ifdefs, rather than 0)?

Quoted 7 lines
Quoted 3 lines

Without any conditionals or comments, it's not clear why x18 is being saved and restored (unless git blame survives, or a comment is added in place of the ifdefs in v6).

True. Clearing the sleep state buffer in cpu_do_resume is also pointless without CONFIG_SHADOW_CALL_STACK, so if the ifdefs are removed, some kind of an explanation is needed there.

I can't imagine the overhead being noticeable, and I certainly value minimizing the testing space. Sticking a comment there should be enough for people hacking on this to understand that this isn't entirely dead code.

Thanks,

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#200 lore raw

PATCH v4 14/17 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +6 -1
index 3fc71106cb2b..945744f16086 100644
@@ -34,5 +34,10 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Restore x18 before returning to instrumented code. */
+	mov	x18, x2
+#endif
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#201 lore raw

PATCH v4 15/17 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#202 lore raw

PATCH v4 16/17 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..17ea3da325e9 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#203 lore raw

PATCH v4 17/17 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 ++++++++++++++++++++++++++
 arch/arm64/include/asm/stacktrace.h  |  4 +++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 28 ++++++++++++++++++++
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 39 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 12 files changed, 137 insertions(+)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 42867174920f..f4c94c5e8012 100644
@@ -66,6 +66,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if GCC_VERSION >= 50000 || CC_IS_CLANG
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -948,6 +949,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/stacktrace.h +4 -0
index 4d9b1f48dc39..b6cf32fb4efe 100644
@@ -68,6 +68,10 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
 
 DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+#endif
+
 static inline bool on_irq_stack(unsigned long sp,
 				struct stack_info *info)
 {
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index 478491f07b4f..b3995329d9e5 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index 214685760e1c..f6762b9ae1e1 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +28 -0
index cf3bd2976e57..1eff08c71403 100644
@@ -172,6 +172,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x20, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -400,6 +413,12 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
@@ -409,6 +428,10 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* x20 is also preserved */
+	mov	x18, x20
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -1155,6 +1178,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..2be977c6496f 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// Set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 71f788cd2b18..5f0aec285848 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -507,6 +508,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +39 -0
new file mode 100644
index 000000000000..6f255072c9a9
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(SCS_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 SCS_GFP, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index dc9fe879c279..cc1938a585d2 100644
@@ -44,6 +44,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -357,6 +358,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#204 lore raw

v5 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer. Because of this, the series includes patches from Ard to remove x18 usage from assembly code.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow, such as function graph tracing, although it may be possible to later change these features to modify the shadow stack instead. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Ard Biesheuvel (3):
arm64/lib: copy_page: avoid x18 register in assembler code
arm64: kvm: stop treating register x18 as caller save
arm64: kernel: avoid x18 in __cpu_soft_restart

Sami Tolvanen (11):
arm64: mm: avoid x18 in idmap_kpti_install_ng_mappings add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
arm64: disable function graph tracing with SCS
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack

 Makefile                             |   6 +
 arch/Kconfig                         |  33 ++++
 arch/arm64/Kconfig                   |   7 +-
 arch/arm64/Makefile                  |   4 +
 arch/arm64/include/asm/scs.h         |  37 ++++
 arch/arm64/include/asm/stacktrace.h  |   4 +
 arch/arm64/include/asm/suspend.h     |   2 +-
 arch/arm64/include/asm/thread_info.h |   3 +
 arch/arm64/kernel/Makefile           |   1 +
 arch/arm64/kernel/asm-offsets.c      |   3 +
 arch/arm64/kernel/cpu-reset.S        |   4 +-
 arch/arm64/kernel/efi-rt-wrapper.S   |   7 +-
 arch/arm64/kernel/entry.S            |  28 +++
 arch/arm64/kernel/head.S             |   9 +
 arch/arm64/kernel/irq.c              |   2 +
 arch/arm64/kernel/process.c          |   2 +
 arch/arm64/kernel/scs.c              |  39 +++++
 arch/arm64/kernel/smp.c              |   4 +
 arch/arm64/kernel/vdso/Makefile      |   2 +-
 arch/arm64/kvm/hyp/Makefile          |   3 +
 arch/arm64/kvm/hyp/entry.S           |  45 ++---
 arch/arm64/lib/copy_page.S           |  38 ++---
 arch/arm64/mm/proc.S                 |  77 +++++----
 drivers/base/node.c                  |   6 +
 fs/proc/meminfo.c                    |   4 +
 include/linux/compiler-clang.h       |   6 +
 include/linux/compiler_types.h       |   4 +
 include/linux/mmzone.h               |   3 +
 include/linux/scs.h                  |  57 +++++++
 init/init_task.c                     |   8 +
 kernel/Makefile                      |   1 +
 kernel/fork.c                        |   9 +
 kernel/sched/core.c                  |   2 +
 kernel/scs.c                         | 246 +++++++++++++++++++++++++++
 mm/page_alloc.c                      |   6 +
 mm/vmstat.c                          |   3 +
 36 files changed, 638 insertions(+), 77 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 26bc672134241a080a83b2ab9aa8abede8d30e1c

Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#205 lore raw

PATCH v5 1/14 arm64: mm: avoid x18 in idmap_kpti_install_ng_mappings

idmap_kpti_install_ng_mappings uses x18 as a temporary register, which will result in a conflict when x18 is reserved. Use x16 and x17 instead where needed.

---
 arch/arm64/mm/proc.S | 63 ++++++++++++++++++++++----------------------
 1 file changed, 32 insertions(+), 31 deletions(-)
arch/arm64/mm/proc.S +32 -31
index a1e0592d1fbc..fdabf40a83c8 100644
@@ -250,15 +250,15 @@ ENTRY(idmap_kpti_install_ng_mappings)
 	/* We're the boot CPU. Wait for the others to catch up */
 	sevl
 1:	wfe
-	ldaxr	w18, [flag_ptr]
-	eor	w18, w18, num_cpus
-	cbnz	w18, 1b
+	ldaxr	w17, [flag_ptr]
+	eor	w17, w17, num_cpus
+	cbnz	w17, 1b
 
 	/* We need to walk swapper, so turn off the MMU. */
 	pre_disable_mmu_workaround
-	mrs	x18, sctlr_el1
-	bic	x18, x18, #SCTLR_ELx_M
-	msr	sctlr_el1, x18
+	mrs	x17, sctlr_el1
+	bic	x17, x17, #SCTLR_ELx_M
+	msr	sctlr_el1, x17
 	isb
 
 	/* Everybody is enjoying the idmap, so we can rewrite swapper. */
@@ -281,9 +281,9 @@ skip_pgd:
 	isb
 
 	/* We're done: fire up the MMU again */
-	mrs	x18, sctlr_el1
-	orr	x18, x18, #SCTLR_ELx_M
-	msr	sctlr_el1, x18
+	mrs	x17, sctlr_el1
+	orr	x17, x17, #SCTLR_ELx_M
+	msr	sctlr_el1, x17
 	isb
 
 	/*
@@ -353,46 +353,47 @@ skip_pte:
 	b.ne	do_pte
 	b	next_pmd
 
+	.unreq	cpu
+	.unreq	num_cpus
+	.unreq	swapper_pa
+	.unreq	cur_pgdp
+	.unreq	end_pgdp
+	.unreq	pgd
+	.unreq	cur_pudp
+	.unreq	end_pudp
+	.unreq	pud
+	.unreq	cur_pmdp
+	.unreq	end_pmdp
+	.unreq	pmd
+	.unreq	cur_ptep
+	.unreq	end_ptep
+	.unreq	pte
+
 	/* Secondary CPUs end up here */
 __idmap_kpti_secondary:
 	/* Uninstall swapper before surgery begins */
-	__idmap_cpu_set_reserved_ttbr1 x18, x17
+	__idmap_cpu_set_reserved_ttbr1 x16, x17
 
 	/* Increment the flag to let the boot CPU we're ready */
-1:	ldxr	w18, [flag_ptr]
-	add	w18, w18, #1
-	stxr	w17, w18, [flag_ptr]
+1:	ldxr	w16, [flag_ptr]
+	add	w16, w16, #1
+	stxr	w17, w16, [flag_ptr]
 	cbnz	w17, 1b
 
 	/* Wait for the boot CPU to finish messing around with swapper */
 	sevl
 1:	wfe
-	ldxr	w18, [flag_ptr]
-	cbnz	w18, 1b
+	ldxr	w16, [flag_ptr]
+	cbnz	w16, 1b
 
 	/* All done, act like nothing happened */
-	offset_ttbr1 swapper_ttb, x18
+	offset_ttbr1 swapper_ttb, x16
 	msr	ttbr1_el1, swapper_ttb
 	isb
 	ret
 
-	.unreq	cpu
-	.unreq	num_cpus
-	.unreq	swapper_pa
 	.unreq	swapper_ttb
 	.unreq	flag_ptr
-	.unreq	cur_pgdp
-	.unreq	end_pgdp
-	.unreq	pgd
-	.unreq	cur_pudp
-	.unreq	end_pudp
-	.unreq	pud
-	.unreq	cur_pmdp
-	.unreq	end_pmdp
-	.unreq	pmd
-	.unreq	cur_ptep
-	.unreq	end_ptep
-	.unreq	pte
 ENDPROC(idmap_kpti_install_ng_mappings)
 	.popsection
 #endif
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#206 lore raw

PATCH v5 2/14 arm64/lib: copy_page: avoid x18 register in assembler code

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

Register x18 will no longer be used as a caller save register in the future, so stop using it in the copy_page() code.

---
 arch/arm64/lib/copy_page.S | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
arch/arm64/lib/copy_page.S +19 -19
index bbb8562396af..290dd3c5266c 100644
@@ -34,45 +34,45 @@ alternative_else_nop_endif
 	ldp	x14, x15, [x1, #96]
 	ldp	x16, x17, [x1, #112]
 
-	mov	x18, #(PAGE_SIZE - 128)
+	add	x0, x0, #256
 	add	x1, x1, #128
 1:
-	subs	x18, x18, #128
+	tst	x0, #(PAGE_SIZE - 1)
 
 alternative_if ARM64_HAS_NO_HW_PREFETCH
 	prfm	pldl1strm, [x1, #384]
 alternative_else_nop_endif
 
-	stnp	x2, x3, [x0]
+	stnp	x2, x3, [x0, #-256]
 	ldp	x2, x3, [x1]
-	stnp	x4, x5, [x0, #16]
+	stnp	x4, x5, [x0, #16 - 256]
 	ldp	x4, x5, [x1, #16]
-	stnp	x6, x7, [x0, #32]
+	stnp	x6, x7, [x0, #32 - 256]
 	ldp	x6, x7, [x1, #32]
-	stnp	x8, x9, [x0, #48]
+	stnp	x8, x9, [x0, #48 - 256]
 	ldp	x8, x9, [x1, #48]
-	stnp	x10, x11, [x0, #64]
+	stnp	x10, x11, [x0, #64 - 256]
 	ldp	x10, x11, [x1, #64]
-	stnp	x12, x13, [x0, #80]
+	stnp	x12, x13, [x0, #80 - 256]
 	ldp	x12, x13, [x1, #80]
-	stnp	x14, x15, [x0, #96]
+	stnp	x14, x15, [x0, #96 - 256]
 	ldp	x14, x15, [x1, #96]
-	stnp	x16, x17, [x0, #112]
+	stnp	x16, x17, [x0, #112 - 256]
 	ldp	x16, x17, [x1, #112]
 
 	add	x0, x0, #128
 	add	x1, x1, #128
 
-	b.gt	1b
+	b.ne	1b
 
-	stnp	x2, x3, [x0]
-	stnp	x4, x5, [x0, #16]
-	stnp	x6, x7, [x0, #32]
-	stnp	x8, x9, [x0, #48]
-	stnp	x10, x11, [x0, #64]
-	stnp	x12, x13, [x0, #80]
-	stnp	x14, x15, [x0, #96]
-	stnp	x16, x17, [x0, #112]
+	stnp	x2, x3, [x0, #-256]
+	stnp	x4, x5, [x0, #16 - 256]
+	stnp	x6, x7, [x0, #32 - 256]
+	stnp	x8, x9, [x0, #48 - 256]
+	stnp	x10, x11, [x0, #64 - 256]
+	stnp	x12, x13, [x0, #80 - 256]
+	stnp	x14, x15, [x0, #96 - 256]
+	stnp	x16, x17, [x0, #112 - 256]
 
 	ret
 ENDPROC(copy_page)
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#207 lore raw

PATCH v5 3/14 arm64: kvm: stop treating register x18 as caller save

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

In preparation of reserving x18, stop treating it as caller save in the KVM guest entry/exit code. Currently, the code assumes there is no need to preserve it for the host, given that it would have been assumed clobbered anyway by the function call to __guest_enter(). Instead, preserve its value and restore it upon return.

[Sami: updated commit message, switched from x18 to x29 for the guest context]
---
 arch/arm64/kvm/hyp/entry.S | 45 ++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 21 deletions(-)
arch/arm64/kvm/hyp/entry.S +24 -21
index e5cc8d66bf53..0c6832ec52b1 100644
@@ -22,7 +22,12 @@
 	.text
 	.pushsection	.hyp.text, "ax"
 
+/*
+ * We treat x18 as callee-saved as the host may use it as a platform
+ * register (e.g. for shadow call stack).
+ */
 .macro save_callee_saved_regs ctxt
+	str	x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
 	stp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
 	stp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
 	stp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
@@ -32,6 +37,8 @@
 .endm
 
 .macro restore_callee_saved_regs ctxt
+	// We require \ctxt is not x18-x28
+	ldr	x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
 	ldp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
 	ldp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
 	ldp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
@@ -48,7 +55,7 @@ ENTRY(__guest_enter)
 	// x0: vcpu
 	// x1: host context
 	// x2-x17: clobbered by macros
-	// x18: guest context
+	// x29: guest context
 
 	// Store the host regs
 	save_callee_saved_regs x1
@@ -67,31 +74,28 @@ alternative_else_nop_endif
 	ret
 
 1:
-	add	x18, x0, #VCPU_CONTEXT
+	add	x29, x0, #VCPU_CONTEXT
 
 	// Macro ptrauth_switch_to_guest format:
 	// 	ptrauth_switch_to_guest(guest cxt, tmp1, tmp2, tmp3)
 	// The below macro to restore guest keys is not implemented in C code
 	// as it may cause Pointer Authentication key signing mismatch errors
 	// when this feature is enabled for kernel code.
-	ptrauth_switch_to_guest x18, x0, x1, x2
+	ptrauth_switch_to_guest x29, x0, x1, x2
 
 	// Restore guest regs x0-x17
-	ldp	x0, x1,   [x18, #CPU_XREG_OFFSET(0)]
-	ldp	x2, x3,   [x18, #CPU_XREG_OFFSET(2)]
-	ldp	x4, x5,   [x18, #CPU_XREG_OFFSET(4)]
-	ldp	x6, x7,   [x18, #CPU_XREG_OFFSET(6)]
-	ldp	x8, x9,   [x18, #CPU_XREG_OFFSET(8)]
-	ldp	x10, x11, [x18, #CPU_XREG_OFFSET(10)]
-	ldp	x12, x13, [x18, #CPU_XREG_OFFSET(12)]
-	ldp	x14, x15, [x18, #CPU_XREG_OFFSET(14)]
-	ldp	x16, x17, [x18, #CPU_XREG_OFFSET(16)]
-
-	// Restore guest regs x19-x29, lr
-	restore_callee_saved_regs x18
-
-	// Restore guest reg x18
-	ldr	x18,      [x18, #CPU_XREG_OFFSET(18)]
+	ldp	x0, x1,   [x29, #CPU_XREG_OFFSET(0)]
+	ldp	x2, x3,   [x29, #CPU_XREG_OFFSET(2)]
+	ldp	x4, x5,   [x29, #CPU_XREG_OFFSET(4)]
+	ldp	x6, x7,   [x29, #CPU_XREG_OFFSET(6)]
+	ldp	x8, x9,   [x29, #CPU_XREG_OFFSET(8)]
+	ldp	x10, x11, [x29, #CPU_XREG_OFFSET(10)]
+	ldp	x12, x13, [x29, #CPU_XREG_OFFSET(12)]
+	ldp	x14, x15, [x29, #CPU_XREG_OFFSET(14)]
+	ldp	x16, x17, [x29, #CPU_XREG_OFFSET(16)]
+
+	// Restore guest regs x18-x29, lr
+	restore_callee_saved_regs x29
 
 	// Do not touch any register after this!
 	eret
@@ -114,7 +118,7 @@ ENTRY(__guest_exit)
 	// Retrieve the guest regs x0-x1 from the stack
 	ldp	x2, x3, [sp], #16	// x0, x1
 
-	// Store the guest regs x0-x1 and x4-x18
+	// Store the guest regs x0-x1 and x4-x17
 	stp	x2, x3,   [x1, #CPU_XREG_OFFSET(0)]
 	stp	x4, x5,   [x1, #CPU_XREG_OFFSET(4)]
 	stp	x6, x7,   [x1, #CPU_XREG_OFFSET(6)]
@@ -123,9 +127,8 @@ ENTRY(__guest_exit)
 	stp	x12, x13, [x1, #CPU_XREG_OFFSET(12)]
 	stp	x14, x15, [x1, #CPU_XREG_OFFSET(14)]
 	stp	x16, x17, [x1, #CPU_XREG_OFFSET(16)]
-	str	x18,      [x1, #CPU_XREG_OFFSET(18)]
 
-	// Store the guest regs x19-x29, lr
+	// Store the guest regs x18-x29, lr
 	save_callee_saved_regs x1
 
 	get_host_ctxt	x2, x3
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#208 lore raw

PATCH v5 4/14 arm64: kernel: avoid x18 in __cpu_soft_restart

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

The code in __cpu_soft_restart() uses x18 as an arbitrary temp register, which will shortly be disallowed. So use x8 instead.

---
 arch/arm64/kernel/cpu-reset.S | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/arm64/kernel/cpu-reset.S +2 -2
index 6ea337d464c4..32c7bf858dd9 100644
@@ -42,11 +42,11 @@ ENTRY(__cpu_soft_restart)
 	mov	x0, #HVC_SOFT_RESTART
 	hvc	#0				// no return
 
-1:	mov	x18, x1				// entry
+1:	mov	x8, x1				// entry
 	mov	x0, x2				// arg0
 	mov	x1, x3				// arg1
 	mov	x2, x4				// arg2
-	br	x18
+	br	x8
 ENDPROC(__cpu_soft_restart)
 
 .popsection
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#209 lore raw

PATCH v5 5/14 add support for Clang's Shadow Call Stack (SCS)

In reply to Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  33 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 313 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index b37d0e8fc61d..7f3a4c5c7dcc 100644
@@ -846,6 +846,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +33 -0
index 5f8a5d84dbbe..5e34cbcd8d6a 100644
@@ -521,6 +521,39 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK_VMAP
+	bool
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index daad787fb795..313dbd44d576 100644
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 55af6931c6ec..6c4266019935 100644
@@ -94,6 +94,7 @@
 #include <linux/livepatch.h>
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -451,6 +452,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -834,6 +837,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -893,6 +898,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index dd05a378631a..6769e27052bf 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6018,6 +6019,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer
#210 lore raw

Re: [PATCH v5 05/14] add support for Clang's Shadow Call Stack (SCS)

Quoted 60 lines · Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  33 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 313 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index b37d0e8fc61d..7f3a4c5c7dcc 100644
@@ -846,6 +846,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +4 -0
index 5f8a5d84dbbe..5e34cbcd8d6a 100644
@@ -521,6 +521,39 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow

arch/Kconfig config STACKPROTECTOR_STRONG · line 537

Quoted 10 lines
arch/Kconfig +10 -0
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK_VMAP
+	bool
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.

The bool needs some display text to make it selectable.

This should probably be below SHADOW_CALL_STACK so that when it shows up in menuconfig it's where you'd expect it to be.

I locally hacked that in, but when building defconfig + SHADOW_CALL_STACK + SHADOW_CALL_STACK_VMAP, the build explodes as below:

| [mark@lakrids:~/src/linux]% usellvm 9.0.0 usekorg 8.1.0 make ARCH=arm64 CROSS_COMPILE=aarch64-linux- CC=clang -j56 -s
| arch/arm64/kernel/scs.c:28:7: error: use of undeclared identifier 'VMALLOC_START'
|                                          VMALLOC_START, VMALLOC_END,
|                                          ^
| arch/arm64/kernel/scs.c:28:22: error: use of undeclared identifier 'VMALLOC_END'
|                                          VMALLOC_START, VMALLOC_END,
|                                                         ^
| arch/arm64/kernel/scs.c:29:7: error: use of undeclared identifier 'SCS_GFP'
|                                          SCS_GFP, PAGE_KERNEL,
|                                          ^
| arch/arm64/kernel/scs.c:29:16: error: use of undeclared identifier 'PAGE_KERNEL'
|                                          SCS_GFP, PAGE_KERNEL,
|                                                   ^
| 4 errors generated.
| scripts/Makefile.build:265: recipe for target 'arch/arm64/kernel/scs.o' failed
| make[2]: *** [arch/arm64/kernel/scs.o] Error 1
| scripts/Makefile.build:509: recipe for target 'arch/arm64/kernel' failed
| make[1]: *** [arch/arm64/kernel] Error 2
| Makefile:1655: recipe for target 'arch/arm64' failed
| make: *** [arch/arm64] Error 2
| make: *** Waiting for unfinished jobs....

Other than that, this largely looks good to me!

Thanks,
Mark.

#211 lore raw

Re: [PATCH v5 05/14] add support for Clang's Shadow Call Stack (SCS)

Quoted 8 lines · Mark Rutland

arch/Kconfig config STACKPROTECTOR_STRONG · line 537

Quoted 7 lines
arch/Kconfig +7 -0
+config SHADOW_CALL_STACK_VMAP
+     bool
+     depends on SHADOW_CALL_STACK
+     help
+       Use virtually mapped shadow call stacks. Selecting this option
+       provides better stack exhaustion protection, but increases per-thread
+       memory consumption as a full page is allocated for each shadow stack.
Quoted 7 lines

The bool needs some display text to make it selectable.

This should probably be below SHADOW_CALL_STACK so that when it shows up in menuconfig it's where you'd expect it to be.

I locally hacked that in, but when building defconfig + SHADOW_CALL_STACK + SHADOW_CALL_STACK_VMAP, the build explodes as below:

Ugh, thanks for pointing this out. I'll fix this in v6.

Sami

#212 lore raw

PATCH v5 6/14 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 296546ffed6c..111e58ec231e 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index bda20282746b..fcb8c1708f9e 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index e3234a4b92ec..4f5774b6f27d 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index ecc3dbad606b..fe17d69d98a7 100644
@@ -5361,6 +5361,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5382,6 +5385,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 6afc892a148a..9fe4afe670fe 100644
@@ -1118,6 +1118,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#213 lore raw

PATCH v5 7/14 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 4f5774b6f27d..a47fae33efdc 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#214 lore raw

PATCH v5 8/14 arm64: disable function graph tracing with SCS

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable the graph tracer when SCS is enabled.

---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index 3f047afb982c..8cda176dad9a 100644
@@ -148,7 +148,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer Reviewed-by on v5 8/14
#215 lore raw

Re: [PATCH v5 08/14] arm64: disable function graph tracing with SCS

Quoted 10 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable the graph tracer when SCS is enabled.

Mark.

Quoted 20 lines
---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index 3f047afb982c..8cda176dad9a 100644
@@ -148,7 +148,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#216 lore raw

PATCH v5 9/14 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 2c0238ce0551..ef76101201b2 100644
@@ -72,6 +72,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#217 lore raw

PATCH v5 10/14 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index fdabf40a83c8..5c8219c55948 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -73,6 +75,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +96,13 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#218 lore raw

Re: [PATCH v5 10/14] arm64: preserve x18 when CPU is suspended

Quoted 5 lines · Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

Re-LGTM

Quoted 61 lines
---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -1
index fdabf40a83c8..5c8219c55948 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
        mrs     x2, tpidr_el0
@@ -73,6 +75,11 @@ alternative_endif
        stp     x8, x9, [x0, #48]
        stp     x10, x11, [x0, #64]
        stp     x12, x13, [x0, #80]
+       /*
+        * Save x18 as it may be used as a platform register, e.g. by shadow
+        * call stack.
+        */
+       str     x18, [x0, #96]
        ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +96,13 @@ ENTRY(cpu_do_resume)
        ldp     x9, x10, [x0, #48]
        ldp     x11, x12, [x0, #64]
        ldp     x13, x14, [x0, #80]
+       /*
+        * Restore x18, as it may be used as a platform register, and clear
+        * the buffer to minimize the risk of exposure when used for shadow
+        * call stack.
+        */
+       ldr     x18, [x0, #96]
+       str     xzr, [x0, #96]
        msr     tpidr_el0, x2
        msr     tpidrro_el0, x3
        msr     contextidr_el1, x4
--

2.24.0.rc1.363.gb1bccd3e3d-goog

Signature · 2 lines
Thanks,
~Nick Desaulniers
reviewer Reviewed-by on v5 10/14
#219 lore raw

Re: [PATCH v5 10/14] arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen
Quoted 15 lines · Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +0 -0
index 8939c87c4dce..0cde2f473971 100644

arch/arm64/include/asm/suspend.h line 5

Quoted 5 lines
arch/arm64/include/asm/suspend.h +1 -1
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13

For a moment I thought this might impact the alignment of the array, but I see cpu_suspend_ctx is force-aligned to 16 bytes anyway, and since commit cabe1c81ea5be983 the only impact would be a performance thing.

Mark.

Quoted 45 lines
arch/arm64/include/asm/suspend.h +0 -0
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index fdabf40a83c8..5c8219c55948 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -73,6 +75,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +96,13 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#220 lore raw

PATCH v5 11/14 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +6 -1
index 3fc71106cb2b..945744f16086 100644
@@ -34,5 +34,10 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Restore x18 before returning to instrumented code. */
+	mov	x18, x2
+#endif
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#221 lore raw

Re: [PATCH v5 11/14] arm64: efi: restore x18 if it was corrupted

Quoted 5 lines · Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

In case you do v6: I think putting the explanation about why this is safe in the existing comment would be best given it is justifying a subtlety of the code rather than the change itself. Ard?

Cheers,
Miguel

#222 lore raw

Re: [PATCH v5 11/14] arm64: efi: restore x18 if it was corrupted

Quoted 12 lines · Miguel Ojeda
Quoted 5 lines · Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

In case you do v6: I think putting the explanation about why this is safe in the existing comment would be best given it is justifying a subtlety of the code rather than the change itself. Ard?

Agreed, but only if you have to respin for other reasons.

#223 lore raw

Re: [PATCH v5 11/14] arm64: efi: restore x18 if it was corrupted

Quoted 11 lines · Ard Biesheuvel
Quoted 8 lines · Miguel Ojeda
Quoted 5 lines · Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

Quoted 6 lines
Quoted 4 lines

In case you do v6: I think putting the explanation about why this is safe in the existing comment would be best given it is justifying a subtlety of the code rather than the change itself. Ard?

Agreed, but only if you have to respin for other reasons.

Sure, sounds good to me. I'll update the comment if other changes are needed.

Sami

#224 lore raw

PATCH v5 12/14 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer Reviewed-by on v5 12/14
#225 lore raw

Re: [PATCH v5 12/14] arm64: vdso: disable Shadow Call Stack

Quoted 6 lines · Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

I gave this a spin, looked at objdump, and found everything in the vDSO was a leaf function. I hacked the code around a bit to force a function call, and I see that just uses x29 and x30 as expected, with nothing touching x18.

Mark.

Quoted 20 lines
---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#226 lore raw

PATCH v5 13/14 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..17ea3da325e9 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer Reviewed-by on v5 13/14
#227 lore raw

Re: [PATCH v5 13/14] arm64: disable SCS for hypervisor code

Quoted 6 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

Looks sound, and objdump confirms that the hyp objects are using x29's frame record, without using x18.

Mark.

Quoted 18 lines
---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..17ea3da325e9 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
#228 lore raw

PATCH v5 14/14 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 ++++++++++++++++++++++++++
 arch/arm64/include/asm/stacktrace.h  |  4 +++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 28 ++++++++++++++++++++
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 39 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 12 files changed, 137 insertions(+)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 8cda176dad9a..76e32d01d759 100644
@@ -66,6 +66,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if GCC_VERSION >= 50000 || CC_IS_CLANG
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -948,6 +949,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/stacktrace.h +4 -0
index 4d9b1f48dc39..b6cf32fb4efe 100644
@@ -68,6 +68,10 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
 
 DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+#endif
+
 static inline bool on_irq_stack(unsigned long sp,
 				struct stack_info *info)
 {
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index 478491f07b4f..b3995329d9e5 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index 214685760e1c..f6762b9ae1e1 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +28 -0
index cf3bd2976e57..1eff08c71403 100644
@@ -172,6 +172,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x20, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -400,6 +413,12 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
@@ -409,6 +428,10 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* x20 is also preserved */
+	mov	x18, x20
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -1155,6 +1178,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..ca561de903d4 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 71f788cd2b18..5f0aec285848 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -507,6 +508,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +39 -0
new file mode 100644
index 000000000000..6f255072c9a9
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(SCS_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 SCS_GFP, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index dc9fe879c279..cc1938a585d2 100644
@@ -44,6 +44,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -357,6 +358,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.24.0.rc1.363.gb1bccd3e3d-goog
reviewer
#229 lore raw

Re: [PATCH v5 14/14] arm64: implement Shadow Call Stack

Quoted 2 lines · Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

Each CPU also has an overflow stack, and two SDEI stacks, which should presumably be given their own SCS. SDEI is effectively a software-NMI, so it should almost certainly have the same treatement as IRQ.

arch/arm64/include/asm/scs.h line 19

Quoted 7 lines
arch/arm64/include/asm/scs.h new file +7 -0
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}

An alternative would be to follow <asm/stack_pointer.h>, and have:

register unsigned long *current_scs_pointer asm ("x18");

static __always_inline void scs_save(struct task_struct *tsk)

{
	task_set_scs(tsk, current_scs_pointer);
}

... which would avoid the need for a temporary register where this is used.

However, given we only use this in cpu_die(), having this as-is should be fine. Maybe the asm volatile is preferable if we use this elsewhere, so that we know we have a consistent snapshot that the compiler can't reload, etc.

[...]

arch/arm64/kernel/entry.S alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 · line 435

Quoted 9 lines
arch/arm64/kernel/entry.S +4 -0
@@ -409,6 +428,10 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* x20 is also preserved */
+	mov	x18, x20
+#endif
 	.endm

Can we please fold this comment into the one above, and have:

/*
* The callee-saved regs (x19-x29) should be preserved between
* irq_stack_entry and irq_stack_exit.
*/
.macro irq_stack_exit
mov sp, x19

#ifdef CONFIG_SHADOW_CALL_STACK
	mov     x18, x20

#endif
.endm

Thanks,
Mark.

#230 lore raw

Re: [PATCH v5 14/14] arm64: implement Shadow Call Stack

Quoted 8 lines · Mark Rutland
Quoted 2 lines · Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

Each CPU also has an overflow stack, and two SDEI stacks, which should presumably be given their own SCS. SDEI is effectively a software-NMI, so it should almost certainly have the same treatement as IRQ.

Makes sense. I'll take a look at adding shadow stacks for the SDEI handler.

Quoted 8 lines

Can we please fold this comment into the one above, and have:

/*
* The callee-saved regs (x19-x29) should be preserved between
* irq_stack_entry and irq_stack_exit.
*/
.macro irq_stack_exit
mov sp, x19

Quoted 4 lines
#ifdef CONFIG_SHADOW_CALL_STACK
        mov     x18, x20

#endif
.endm

Sure, I'll change this in the next version.

Sami

#231 lore raw

Re: [PATCH v5 14/14] arm64: implement Shadow Call Stack

Quoted 11 lines · Sami Tolvanen
Quoted 8 lines · Mark Rutland
Quoted 2 lines · Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

Each CPU also has an overflow stack, and two SDEI stacks, which should presumably be given their own SCS. SDEI is effectively a software-NMI, so it should almost certainly have the same treatement as IRQ.

Makes sense. I'll take a look at adding shadow stacks for the SDEI handler.

Mark, I wrote a preliminary patch for adding SDEI shadow stacks, but turns out I don't really have a way to test the SDEI code. Does the approach below look sane to you?

Sami

---
 arch/arm64/include/asm/scs.h        |   2 +
 arch/arm64/include/asm/stacktrace.h |   4 --
 arch/arm64/kernel/entry.S           |  14 +++-
 arch/arm64/kernel/scs.c             | 106 +++++++++++++++++++++++-----
 arch/arm64/kernel/sdei.c            |   7 ++
 5 files changed, 112 insertions(+), 21 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/include/asm/stacktrace.h +0 -4
index b6cf32fb4efe..4d9b1f48dc39 100644
@@ -68,10 +68,6 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
 
 DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
 
-#ifdef CONFIG_SHADOW_CALL_STACK
-DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
-#endif
-
 static inline bool on_irq_stack(unsigned long sp,
 				struct stack_info *info)
 {
arch/arm64/kernel/entry.S +13 -1
index 5a02b61fc3e6..ac9dfb3da440 100644
@@ -1309,13 +1309,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1325,6 +1328,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index ea94cf8f9dc6..3e85017a9c8b 100644
@@ -12,6 +12,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -161,6 +162,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
#232 lore raw
In reply to Sami Tolvanen
Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information

Will, Catalin, Mark,

What's the next step here? I *think* all the comments have been addressed. Is it possible to land this via the arm tree for v5.5?

Thanks!

Signature · 1 line
Kees Cook
maintainer
#233 lore raw
Quoted 9 lines · Kees Cook
Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information

Will, Catalin, Mark,

What's the next step here? I *think* all the comments have been addressed. Is it possible to land this via the arm tree for v5.5?

I was planning to queue this for 5.6, given that I'd really like it to spend some quality time in linux-next.

Will

#234 lore raw
Quoted 6 lines · Will Deacon
Quoted 5 lines · Kees Cook
Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information

Quoted 7 lines
Quoted 4 lines

Will, Catalin, Mark,

What's the next step here? I *think* all the comments have been addressed. Is it possible to land this via the arm tree for v5.5?

I was planning to queue this for 5.6, given that I'd really like it to spend some quality time in linux-next.

Sounds fine to me; I just wanted to have an idea what to expect. :) Thanks!

Signature · 1 line
Kees Cook
reviewer
#235 lore raw
In reply to Kees Cook
Quoted 9 lines · Kees Cook
Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information

Will, Catalin, Mark,

What's the next step here? I *think* all the comments have been addressed.

I'm hoping to look over the remaining bits in the next week or so, and to throw my test boxes at this shortly.

Thanks,
Mark.

#236 lore raw

v6 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer. Because of this, the series includes patches from Ard to remove x18 usage from assembly code.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow, such as function graph tracing, although it may be possible to later change these features to modify the shadow stack instead. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Ard Biesheuvel (3):
arm64/lib: copy_page: avoid x18 register in assembler code
arm64: kvm: stop treating register x18 as caller save
arm64: kernel: avoid x18 in __cpu_soft_restart

Sami Tolvanen (12):
arm64: mm: avoid x18 in idmap_kpti_install_ng_mappings add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
arm64: disable function graph tracing with SCS
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI

 Makefile                             |   6 +
 arch/Kconfig                         |  34 ++++
 arch/arm64/Kconfig                   |   7 +-
 arch/arm64/Makefile                  |   4 +
 arch/arm64/include/asm/scs.h         |  39 +++++
 arch/arm64/include/asm/suspend.h     |   2 +-
 arch/arm64/include/asm/thread_info.h |   3 +
 arch/arm64/kernel/Makefile           |   1 +
 arch/arm64/kernel/asm-offsets.c      |   3 +
 arch/arm64/kernel/cpu-reset.S        |   4 +-
 arch/arm64/kernel/efi-rt-wrapper.S   |  11 +-
 arch/arm64/kernel/entry.S            |  45 ++++-
 arch/arm64/kernel/head.S             |   9 +
 arch/arm64/kernel/irq.c              |   2 +
 arch/arm64/kernel/process.c          |   2 +
 arch/arm64/kernel/scs.c              | 114 +++++++++++++
 arch/arm64/kernel/sdei.c             |   7 +
 arch/arm64/kernel/smp.c              |   4 +
 arch/arm64/kernel/vdso/Makefile      |   2 +-
 arch/arm64/kvm/hyp/Makefile          |   3 +
 arch/arm64/kvm/hyp/entry.S           |  45 ++---
 arch/arm64/lib/copy_page.S           |  38 ++---
 arch/arm64/mm/proc.S                 |  77 +++++----
 drivers/base/node.c                  |   6 +
 fs/proc/meminfo.c                    |   4 +
 include/linux/compiler-clang.h       |   6 +
 include/linux/compiler_types.h       |   4 +
 include/linux/mmzone.h               |   3 +
 include/linux/scs.h                  |  57 +++++++
 init/init_task.c                     |   8 +
 kernel/Makefile                      |   1 +
 kernel/fork.c                        |   9 +
 kernel/sched/core.c                  |   2 +
 kernel/scs.c                         | 246 +++++++++++++++++++++++++++
 mm/page_alloc.c                      |   6 +
 mm/vmstat.c                          |   3 +
 36 files changed, 737 insertions(+), 80 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 3cf2890f29ab6fe491361761df558ef9191cb468

Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#237 lore raw

PATCH v6 1/15 arm64: mm: avoid x18 in idmap_kpti_install_ng_mappings

idmap_kpti_install_ng_mappings uses x18 as a temporary register, which will result in a conflict when x18 is reserved. Use x16 and x17 instead where needed.

---
 arch/arm64/mm/proc.S | 63 ++++++++++++++++++++++----------------------
 1 file changed, 32 insertions(+), 31 deletions(-)
arch/arm64/mm/proc.S +32 -31
index a1e0592d1fbc..fdabf40a83c8 100644
@@ -250,15 +250,15 @@ ENTRY(idmap_kpti_install_ng_mappings)
 	/* We're the boot CPU. Wait for the others to catch up */
 	sevl
 1:	wfe
-	ldaxr	w18, [flag_ptr]
-	eor	w18, w18, num_cpus
-	cbnz	w18, 1b
+	ldaxr	w17, [flag_ptr]
+	eor	w17, w17, num_cpus
+	cbnz	w17, 1b
 
 	/* We need to walk swapper, so turn off the MMU. */
 	pre_disable_mmu_workaround
-	mrs	x18, sctlr_el1
-	bic	x18, x18, #SCTLR_ELx_M
-	msr	sctlr_el1, x18
+	mrs	x17, sctlr_el1
+	bic	x17, x17, #SCTLR_ELx_M
+	msr	sctlr_el1, x17
 	isb
 
 	/* Everybody is enjoying the idmap, so we can rewrite swapper. */
@@ -281,9 +281,9 @@ skip_pgd:
 	isb
 
 	/* We're done: fire up the MMU again */
-	mrs	x18, sctlr_el1
-	orr	x18, x18, #SCTLR_ELx_M
-	msr	sctlr_el1, x18
+	mrs	x17, sctlr_el1
+	orr	x17, x17, #SCTLR_ELx_M
+	msr	sctlr_el1, x17
 	isb
 
 	/*
@@ -353,46 +353,47 @@ skip_pte:
 	b.ne	do_pte
 	b	next_pmd
 
+	.unreq	cpu
+	.unreq	num_cpus
+	.unreq	swapper_pa
+	.unreq	cur_pgdp
+	.unreq	end_pgdp
+	.unreq	pgd
+	.unreq	cur_pudp
+	.unreq	end_pudp
+	.unreq	pud
+	.unreq	cur_pmdp
+	.unreq	end_pmdp
+	.unreq	pmd
+	.unreq	cur_ptep
+	.unreq	end_ptep
+	.unreq	pte
+
 	/* Secondary CPUs end up here */
 __idmap_kpti_secondary:
 	/* Uninstall swapper before surgery begins */
-	__idmap_cpu_set_reserved_ttbr1 x18, x17
+	__idmap_cpu_set_reserved_ttbr1 x16, x17
 
 	/* Increment the flag to let the boot CPU we're ready */
-1:	ldxr	w18, [flag_ptr]
-	add	w18, w18, #1
-	stxr	w17, w18, [flag_ptr]
+1:	ldxr	w16, [flag_ptr]
+	add	w16, w16, #1
+	stxr	w17, w16, [flag_ptr]
 	cbnz	w17, 1b
 
 	/* Wait for the boot CPU to finish messing around with swapper */
 	sevl
 1:	wfe
-	ldxr	w18, [flag_ptr]
-	cbnz	w18, 1b
+	ldxr	w16, [flag_ptr]
+	cbnz	w16, 1b
 
 	/* All done, act like nothing happened */
-	offset_ttbr1 swapper_ttb, x18
+	offset_ttbr1 swapper_ttb, x16
 	msr	ttbr1_el1, swapper_ttb
 	isb
 	ret
 
-	.unreq	cpu
-	.unreq	num_cpus
-	.unreq	swapper_pa
 	.unreq	swapper_ttb
 	.unreq	flag_ptr
-	.unreq	cur_pgdp
-	.unreq	end_pgdp
-	.unreq	pgd
-	.unreq	cur_pudp
-	.unreq	end_pudp
-	.unreq	pud
-	.unreq	cur_pmdp
-	.unreq	end_pmdp
-	.unreq	pmd
-	.unreq	cur_ptep
-	.unreq	end_ptep
-	.unreq	pte
 ENDPROC(idmap_kpti_install_ng_mappings)
 	.popsection
 #endif
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#238 lore raw

PATCH v6 2/15 arm64/lib: copy_page: avoid x18 register in assembler code

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

Register x18 will no longer be used as a caller save register in the future, so stop using it in the copy_page() code.

---
 arch/arm64/lib/copy_page.S | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
arch/arm64/lib/copy_page.S +19 -19
index bbb8562396af..290dd3c5266c 100644
@@ -34,45 +34,45 @@ alternative_else_nop_endif
 	ldp	x14, x15, [x1, #96]
 	ldp	x16, x17, [x1, #112]
 
-	mov	x18, #(PAGE_SIZE - 128)
+	add	x0, x0, #256
 	add	x1, x1, #128
 1:
-	subs	x18, x18, #128
+	tst	x0, #(PAGE_SIZE - 1)
 
 alternative_if ARM64_HAS_NO_HW_PREFETCH
 	prfm	pldl1strm, [x1, #384]
 alternative_else_nop_endif
 
-	stnp	x2, x3, [x0]
+	stnp	x2, x3, [x0, #-256]
 	ldp	x2, x3, [x1]
-	stnp	x4, x5, [x0, #16]
+	stnp	x4, x5, [x0, #16 - 256]
 	ldp	x4, x5, [x1, #16]
-	stnp	x6, x7, [x0, #32]
+	stnp	x6, x7, [x0, #32 - 256]
 	ldp	x6, x7, [x1, #32]
-	stnp	x8, x9, [x0, #48]
+	stnp	x8, x9, [x0, #48 - 256]
 	ldp	x8, x9, [x1, #48]
-	stnp	x10, x11, [x0, #64]
+	stnp	x10, x11, [x0, #64 - 256]
 	ldp	x10, x11, [x1, #64]
-	stnp	x12, x13, [x0, #80]
+	stnp	x12, x13, [x0, #80 - 256]
 	ldp	x12, x13, [x1, #80]
-	stnp	x14, x15, [x0, #96]
+	stnp	x14, x15, [x0, #96 - 256]
 	ldp	x14, x15, [x1, #96]
-	stnp	x16, x17, [x0, #112]
+	stnp	x16, x17, [x0, #112 - 256]
 	ldp	x16, x17, [x1, #112]
 
 	add	x0, x0, #128
 	add	x1, x1, #128
 
-	b.gt	1b
+	b.ne	1b
 
-	stnp	x2, x3, [x0]
-	stnp	x4, x5, [x0, #16]
-	stnp	x6, x7, [x0, #32]
-	stnp	x8, x9, [x0, #48]
-	stnp	x10, x11, [x0, #64]
-	stnp	x12, x13, [x0, #80]
-	stnp	x14, x15, [x0, #96]
-	stnp	x16, x17, [x0, #112]
+	stnp	x2, x3, [x0, #-256]
+	stnp	x4, x5, [x0, #16 - 256]
+	stnp	x6, x7, [x0, #32 - 256]
+	stnp	x8, x9, [x0, #48 - 256]
+	stnp	x10, x11, [x0, #64 - 256]
+	stnp	x12, x13, [x0, #80 - 256]
+	stnp	x14, x15, [x0, #96 - 256]
+	stnp	x16, x17, [x0, #112 - 256]
 
 	ret
 ENDPROC(copy_page)
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#239 lore raw

PATCH v6 3/15 arm64: kvm: stop treating register x18 as caller save

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

In preparation of reserving x18, stop treating it as caller save in the KVM guest entry/exit code. Currently, the code assumes there is no need to preserve it for the host, given that it would have been assumed clobbered anyway by the function call to __guest_enter(). Instead, preserve its value and restore it upon return.

[Sami: updated commit message, switched from x18 to x29 for the guest context]
---
 arch/arm64/kvm/hyp/entry.S | 45 ++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 21 deletions(-)
arch/arm64/kvm/hyp/entry.S +24 -21
index e5cc8d66bf53..0c6832ec52b1 100644
@@ -22,7 +22,12 @@
 	.text
 	.pushsection	.hyp.text, "ax"
 
+/*
+ * We treat x18 as callee-saved as the host may use it as a platform
+ * register (e.g. for shadow call stack).
+ */
 .macro save_callee_saved_regs ctxt
+	str	x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
 	stp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
 	stp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
 	stp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
@@ -32,6 +37,8 @@
 .endm
 
 .macro restore_callee_saved_regs ctxt
+	// We require \ctxt is not x18-x28
+	ldr	x18,      [\ctxt, #CPU_XREG_OFFSET(18)]
 	ldp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
 	ldp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
 	ldp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
@@ -48,7 +55,7 @@ ENTRY(__guest_enter)
 	// x0: vcpu
 	// x1: host context
 	// x2-x17: clobbered by macros
-	// x18: guest context
+	// x29: guest context
 
 	// Store the host regs
 	save_callee_saved_regs x1
@@ -67,31 +74,28 @@ alternative_else_nop_endif
 	ret
 
 1:
-	add	x18, x0, #VCPU_CONTEXT
+	add	x29, x0, #VCPU_CONTEXT
 
 	// Macro ptrauth_switch_to_guest format:
 	// 	ptrauth_switch_to_guest(guest cxt, tmp1, tmp2, tmp3)
 	// The below macro to restore guest keys is not implemented in C code
 	// as it may cause Pointer Authentication key signing mismatch errors
 	// when this feature is enabled for kernel code.
-	ptrauth_switch_to_guest x18, x0, x1, x2
+	ptrauth_switch_to_guest x29, x0, x1, x2
 
 	// Restore guest regs x0-x17
-	ldp	x0, x1,   [x18, #CPU_XREG_OFFSET(0)]
-	ldp	x2, x3,   [x18, #CPU_XREG_OFFSET(2)]
-	ldp	x4, x5,   [x18, #CPU_XREG_OFFSET(4)]
-	ldp	x6, x7,   [x18, #CPU_XREG_OFFSET(6)]
-	ldp	x8, x9,   [x18, #CPU_XREG_OFFSET(8)]
-	ldp	x10, x11, [x18, #CPU_XREG_OFFSET(10)]
-	ldp	x12, x13, [x18, #CPU_XREG_OFFSET(12)]
-	ldp	x14, x15, [x18, #CPU_XREG_OFFSET(14)]
-	ldp	x16, x17, [x18, #CPU_XREG_OFFSET(16)]
-
-	// Restore guest regs x19-x29, lr
-	restore_callee_saved_regs x18
-
-	// Restore guest reg x18
-	ldr	x18,      [x18, #CPU_XREG_OFFSET(18)]
+	ldp	x0, x1,   [x29, #CPU_XREG_OFFSET(0)]
+	ldp	x2, x3,   [x29, #CPU_XREG_OFFSET(2)]
+	ldp	x4, x5,   [x29, #CPU_XREG_OFFSET(4)]
+	ldp	x6, x7,   [x29, #CPU_XREG_OFFSET(6)]
+	ldp	x8, x9,   [x29, #CPU_XREG_OFFSET(8)]
+	ldp	x10, x11, [x29, #CPU_XREG_OFFSET(10)]
+	ldp	x12, x13, [x29, #CPU_XREG_OFFSET(12)]
+	ldp	x14, x15, [x29, #CPU_XREG_OFFSET(14)]
+	ldp	x16, x17, [x29, #CPU_XREG_OFFSET(16)]
+
+	// Restore guest regs x18-x29, lr
+	restore_callee_saved_regs x29
 
 	// Do not touch any register after this!
 	eret
@@ -114,7 +118,7 @@ ENTRY(__guest_exit)
 	// Retrieve the guest regs x0-x1 from the stack
 	ldp	x2, x3, [sp], #16	// x0, x1
 
-	// Store the guest regs x0-x1 and x4-x18
+	// Store the guest regs x0-x1 and x4-x17
 	stp	x2, x3,   [x1, #CPU_XREG_OFFSET(0)]
 	stp	x4, x5,   [x1, #CPU_XREG_OFFSET(4)]
 	stp	x6, x7,   [x1, #CPU_XREG_OFFSET(6)]
@@ -123,9 +127,8 @@ ENTRY(__guest_exit)
 	stp	x12, x13, [x1, #CPU_XREG_OFFSET(12)]
 	stp	x14, x15, [x1, #CPU_XREG_OFFSET(14)]
 	stp	x16, x17, [x1, #CPU_XREG_OFFSET(16)]
-	str	x18,      [x1, #CPU_XREG_OFFSET(18)]
 
-	// Store the guest regs x19-x29, lr
+	// Store the guest regs x18-x29, lr
 	save_callee_saved_regs x1
 
 	get_host_ctxt	x2, x3
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#240 lore raw

PATCH v6 4/15 arm64: kernel: avoid x18 in __cpu_soft_restart

In reply to Sami Tolvanen

From: Ard Biesheuvel <[email protected]>

The code in __cpu_soft_restart() uses x18 as an arbitrary temp register, which will shortly be disallowed. So use x8 instead.

---
 arch/arm64/kernel/cpu-reset.S | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/arm64/kernel/cpu-reset.S +2 -2
index 6ea337d464c4..32c7bf858dd9 100644
@@ -42,11 +42,11 @@ ENTRY(__cpu_soft_restart)
 	mov	x0, #HVC_SOFT_RESTART
 	hvc	#0				// no return
 
-1:	mov	x18, x1				// entry
+1:	mov	x8, x1				// entry
 	mov	x0, x2				// arg0
 	mov	x1, x3				// arg1
 	mov	x2, x4				// arg2
-	br	x18
+	br	x8
 ENDPROC(__cpu_soft_restart)
 
 .popsection
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#241 lore raw

PATCH v6 5/15 add support for Clang's Shadow Call Stack (SCS)

In reply to Sami Tolvanen

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 999a197d67d2..1b76c1ea2a02 100644
@@ -845,6 +845,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 48b5e103bdb0..1b16aa9a3fe5 100644
@@ -521,6 +521,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index f2cc0d118a0b..06231f936a7a 100644
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 2508a4f238a3..bf52528f9b36 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -454,6 +455,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -837,6 +840,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -896,6 +901,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 90e4b00ace89..a181c536e12e 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6038,6 +6039,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#242 lore raw

PATCH v6 6/15 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 98a31bafc8a2..874a8b428438 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 89d8ff06c9ce..dfeb8a5a7f7e 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 4785a8a2040e..6073556fc99a 100644
@@ -5365,6 +5365,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5387,6 +5390,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 78d53378db99..d0650391c8c1 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#243 lore raw

PATCH v6 7/15 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#244 lore raw

PATCH v6 8/15 arm64: disable function graph tracing with SCS

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable the graph tracer when SCS is enabled.

---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +1 -1
index b1b4476ddb83..49e5f94ff4af 100644
@@ -149,7 +149,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#245 lore raw

Re: [PATCH v6 08/15] arm64: disable function graph tracing with SCS

Quoted 20 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable the graph tracer when SCS is enabled.

---
 arch/arm64/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/Kconfig +0 -0
index b1b4476ddb83..49e5f94ff4af 100644

arch/arm64/Kconfig config ARM64 · line 155

Quoted 8 lines
arch/arm64/Kconfig +1 -1
@@ -149,7 +149,7 @@ config ARM64
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_ERROR_INJECTION
-	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING

I think this is the wrong way around, as we support the graph tracer today and so I think SHADOW_CALL_STACK should depend on !GRAPH_TRACER and possibly even EXPERT until this is resolved.

Will

#246 lore raw

Re: [PATCH v6 08/15] arm64: disable function graph tracing with SCS

Quoted 11 lines · Will Deacon
Quoted 9 lines · Sami Tolvanen
arch/arm64/Kconfig +0 -1
index b1b4476ddb83..49e5f94ff4af 100644
@@ -149,7 +149,7 @@ config ARM64
      select HAVE_FTRACE_MCOUNT_RECORD
      select HAVE_FUNCTION_TRACER
      select HAVE_FUNCTION_ERROR_INJECTION
-     select HAVE_FUNCTION_GRAPH_TRACER
Quoted 8 lines

arch/arm64/Kconfig config ARM64 · line 155

Quoted 4 lines
arch/arm64/Kconfig +1 -0
+     select HAVE_FUNCTION_GRAPH_TRACER if !SHADOW_CALL_STACK
      select HAVE_GCC_PLUGINS
      select HAVE_HW_BREAKPOINT if PERF_EVENTS
      select HAVE_IRQ_TIME_ACCOUNTING

I think this is the wrong way around, as we support the graph tracer today and so I think SHADOW_CALL_STACK should depend on !GRAPH_TRACER and possibly even EXPERT until this is resolved.

Sure, sounds reasonable. I'll change this in the next version.

Sami

#247 lore raw

PATCH v6 9/15 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index 1fbe24d4fdb6..e69736fc1106 100644
@@ -72,6 +72,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer Acked-by on v6 9/15
#248 lore raw

Re: [PATCH v6 09/15] arm64: reserve x18 from general allocation with SCS

Quoted 11 lines · Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)

Will

#249 lore raw

PATCH v6 10/15 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index fdabf40a83c8..5c8219c55948 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -73,6 +75,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -89,6 +96,13 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer Acked-by on v6 10/15
#250 lore raw

Re: [PATCH v6 10/15] arm64: preserve x18 when CPU is suspended

Quoted 50 lines · Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +7 -0
index fdabf40a83c8..5c8219c55948 100644
@@ -49,6 +49,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -73,6 +75,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 ENDPROC(cpu_do_suspend)
 

arch/arm64/mm/proc.S ENTRY(cpu_do_resume) · line 105

Quoted 10 lines
arch/arm64/mm/proc.S +7 -0
@@ -89,6 +96,13 @@ ENTRY(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]

Mumble, mumble, spectre-v4.

But I think it's fairly hopeless trying to fix that everywhere it crops up, so:

Will

#251 lore raw

PATCH v6 11/15 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..62f0260f5c17 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/*
+	 * Restore x18 before returning to instrumented code. This is
+	 * safe because the wrapper is called with preemption disabled and
+	 * a separate shadow stack is used for interrupts.
+	 */
+	mov	x18, x2
+#endif
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#252 lore raw

Re: [PATCH v6 11/15] arm64: efi: restore x18 if it was corrupted

Quoted 19 lines · Sami Tolvanen

If we detect a corrupted x18 and SCS is enabled, restore the register before jumping back to instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +0 -0
index 3fc71106cb2b..62f0260f5c17 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret

arch/arm64/kernel/efi-rt-wrapper.S ENTRY(__efi_rt_asm_wrapper) · line 45

Quoted 10 lines
arch/arm64/kernel/efi-rt-wrapper.S +9 -1
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/*
+	 * Restore x18 before returning to instrumented code. This is
+	 * safe because the wrapper is called with preemption disabled and
+	 * a separate shadow stack is used for interrupts.
+	 */
+	mov	x18, x2
+#endif

Why not restore it regardless of CONFIG_SHADOW_CALL_STACK?

Will

#253 lore raw

Re: [PATCH v6 11/15] arm64: efi: restore x18 if it was corrupted

Quoted 14 lines · Will Deacon

arch/arm64/kernel/efi-rt-wrapper.S ENTRY(__efi_rt_asm_wrapper) · line 45

Quoted 10 lines · Sami Tolvanen

-0: b efi_handle_corrupted_x18 // tail call
+0:

arch/arm64/kernel/efi-rt-wrapper.S +8 -0
+#ifdef CONFIG_SHADOW_CALL_STACK
+     /*
+      * Restore x18 before returning to instrumented code. This is
+      * safe because the wrapper is called with preemption disabled and
+      * a separate shadow stack is used for interrupts.
+      */
+     mov     x18, x2
+#endif

Why not restore it regardless of CONFIG_SHADOW_CALL_STACK?

The ifdefs are here only because restoring the register without SCS isn't actually necessary, but I'm fine with dropping them (and editing the comment) in the next version if you prefer.

Sami

#254 lore raw

PATCH v6 12/15 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#255 lore raw

Re: [PATCH v6 12/15] arm64: vdso: disable Shadow Call Stack

Quoted 16 lines · Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +0 -0
index dd2514bb1511..a87a4f11724e 100644

arch/arm64/kernel/vdso/Makefile ccflags-y += -DDISABLE_BRANCH_PROFILING · line 28

Quoted 5 lines
arch/arm64/kernel/vdso/Makefile +1 -1
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)

Should we be removing -ffixed-x18 too, or does that not propagate here anyway?

Will

#256 lore raw

Re: [PATCH v6 12/15] arm64: vdso: disable Shadow Call Stack

Quoted 2 lines · Will Deacon

Should we be removing -ffixed-x18 too, or does that not propagate here anyway?

No, we shouldn't touch -ffixed-x18 here. The vDSO is always built with x18 reserved since commit 98cd3c3f83fbb ("arm64: vdso: Build vDSO with -ffixed-x18").

Sami

maintainer Acked-by
#257 lore raw

Re: [PATCH v6 12/15] arm64: vdso: disable Shadow Call Stack

Quoted 7 lines · Sami Tolvanen
Quoted 2 lines · Will Deacon

Should we be removing -ffixed-x18 too, or does that not propagate here anyway?

No, we shouldn't touch -ffixed-x18 here. The vDSO is always built with x18 reserved since commit 98cd3c3f83fbb ("arm64: vdso: Build vDSO with -ffixed-x18").

Thanks, in which case:

Will

#258 lore raw

PATCH v6 13/15 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..17ea3da325e9 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#259 lore raw

Re: [PATCH v6 13/15] arm64: disable SCS for hypervisor code

Quoted 16 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +0 -0
index ea710f674cb6..17ea3da325e9 100644

arch/arm64/kvm/hyp/Makefile GCOV_PROFILE := n · line 33

Quoted 6 lines
arch/arm64/kvm/hyp/Makefile +3 -0
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))

Same comment as for the vDSO; can we remove the -ffixed-x18 as well?

Will

#260 lore raw

Re: [PATCH v6 13/15] arm64: disable SCS for hypervisor code

Quoted 13 lines · Will Deacon
Quoted 11 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
Quoted 13 lines

arch/arm64/kvm/hyp/Makefile GCOV_PROFILE := n · line 33

Quoted 11 lines
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..17ea3da325e9 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE        := n
 KASAN_SANITIZE       := n
 UBSAN_SANITIZE       := n
 KCOV_INSTRUMENT      := n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))

Same comment as for the vDSO; can we remove the -ffixed-x18 as well?

Sure, I don't see why not. I'll change this in the next version.

Sami

#261 lore raw

PATCH v6 14/15 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 31 +++++++++++++++++++--
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 135 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 49e5f94ff4af..073b19db23f2 100644
@@ -65,6 +65,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -994,6 +995,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index fc6488660f64..08fafc4da2cf 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index a5bdce8af65b..d485dc5cd196 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +29 -2
index 583f71abbe98..7aa2d366b2df 100644
@@ -172,6 +172,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -280,6 +284,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -385,6 +395,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x20, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -402,15 +415,24 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x20
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -894,6 +916,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..ca561de903d4 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 71f788cd2b18..5f0aec285848 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -507,6 +508,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index ab149bcc3dc7..416cf4da6f0c 100644
@@ -44,6 +44,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -356,6 +357,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#262 lore raw

Re: [PATCH v6 14/15] arm64: implement Shadow Call Stack

Quoted 20 lines · Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 31 +++++++++++++++++++--
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 135 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c

[...]

Quoted 29 lines
arch/arm64/kernel/entry.S +10 -0
index 583f71abbe98..7aa2d366b2df 100644
@@ -172,6 +172,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -280,6 +284,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR

arch/arm64/kernel/entry.S alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 · line 400

Quoted 6 lines
arch/arm64/kernel/entry.S +3 -0
@@ -385,6 +395,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x20, x18		// preserve the original shadow stack
+#endif

Hmm, not sure about corrupting x20 here. Doesn't it hold the PMR value from kernel_entry?

Rest of the patch looks ok, but I'll do a proper review when it's closer to being merged as we've got a bunch of other entry changes in the pipeline.

Will

#263 lore raw

Re: [PATCH v6 14/15] arm64: implement Shadow Call Stack

Quoted 8 lines · Will Deacon

arch/arm64/kernel/entry.S alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 · line 400

Quoted 5 lines

.macro irq_stack_entry
mov x19, sp // preserve the original sp

arch/arm64/kernel/entry.S +3 -0
+#ifdef CONFIG_SHADOW_CALL_STACK
+     mov     x20, x18                // preserve the original shadow stack
+#endif

Hmm, not sure about corrupting x20 here. Doesn't it hold the PMR value from kernel_entry?

You're right, and it's used in el1_irq after irq_handler if CONFIG_ARM64_PSEUDO_NMI is enabled. Thanks for pointing this out. Looks like one of x24-x29 should be safe here, and the comment needs to be updated to explain why x20-x23 shouldn't be corrupted.

Sami

#264 lore raw

PATCH v6 15/15 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index 7aa2d366b2df..9327c3d21b64 100644
@@ -1048,13 +1048,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1064,6 +1067,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
maintainer
#265 lore raw

Re: [PATCH v6 15/15] arm64: scs: add shadow stacks for SDEI

[+James, since this needs his Ack before it can be merged]

Quoted 224 lines · Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index 7aa2d366b2df..9327c3d21b64 100644
@@ -1048,13 +1048,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1064,6 +1067,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16
index eaadf5430baa..dddb7c56518b 100644
@@ -10,31 +10,105 @@
 #include <asm/pgtable.h>
 #include <asm/scs.h>
 
-DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+#define DECLARE_SCS(name)						\
+	DECLARE_PER_CPU(unsigned long *, name ## _ptr);			\
+	DECLARE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)
 
-#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
-DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
-	__aligned(SCS_SIZE);
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+#define DEFINE_SCS(name)						\
+	DEFINE_PER_CPU(unsigned long *, name ## _ptr)
+#else
+/* Allocate a static per-CPU shadow stack */
+#define DEFINE_SCS(name)						\
+	DEFINE_PER_CPU(unsigned long *, name ## _ptr);			\
+	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
+		__aligned(SCS_SIZE)
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+
+DECLARE_SCS(irq_shadow_call_stack);
+DECLARE_SCS(sdei_shadow_call_stack_normal);
+DECLARE_SCS(sdei_shadow_call_stack_critical);
+
+DEFINE_SCS(irq_shadow_call_stack);
+#ifdef CONFIG_ARM_SDE_INTERFACE
+DEFINE_SCS(sdei_shadow_call_stack_normal);
+DEFINE_SCS(sdei_shadow_call_stack_critical);
 #endif
 
+static int scs_alloc_percpu(unsigned long * __percpu *ptr, int cpu)
+{
+	unsigned long *p;
+
+	p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+				 VMALLOC_START, VMALLOC_END,
+				 GFP_SCS, PAGE_KERNEL,
+				 0, cpu_to_node(cpu),
+				 __builtin_return_address(0));
+
+	if (!p)
+		return -ENOMEM;
+	per_cpu(*ptr, cpu) = p;
+
+	return 0;
+}
+
+static void scs_free_percpu(unsigned long * __percpu *ptr, int cpu)
+{
+	unsigned long *p = per_cpu(*ptr, cpu);
+
+	if (p) {
+		per_cpu(*ptr, cpu) = NULL;
+		vfree(p);
+	}
+}
+
+static void scs_free_sdei(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		scs_free_percpu(&sdei_shadow_call_stack_normal_ptr, cpu);
+		scs_free_percpu(&sdei_shadow_call_stack_critical_ptr, cpu);
+	}
+}
+
 void scs_init_irq(void)
 {
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
-#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
-		unsigned long *p;
+		if (IS_ENABLED(CONFIG_SHADOW_CALL_STACK_VMAP))
+			WARN_ON(scs_alloc_percpu(&irq_shadow_call_stack_ptr,
+						 cpu));
+		else
+			per_cpu(irq_shadow_call_stack_ptr, cpu) =
+				per_cpu(irq_shadow_call_stack, cpu);
+	}
+}
 
-		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
-					 VMALLOC_START, VMALLOC_END,
-					 GFP_SCS, PAGE_KERNEL,
-					 0, cpu_to_node(cpu),
-					 __builtin_return_address(0));
+int scs_init_sdei(void)
+{
+	int cpu;
 
-		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
-#else
-		per_cpu(irq_shadow_call_stack_ptr, cpu) =
-			per_cpu(irq_shadow_call_stack, cpu);
-#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
+		return 0;
+
+	for_each_possible_cpu(cpu) {
+		if (IS_ENABLED(CONFIG_SHADOW_CALL_STACK_VMAP)) {
+			if (scs_alloc_percpu(
+				&sdei_shadow_call_stack_normal_ptr, cpu) ||
+			    scs_alloc_percpu(
+				&sdei_shadow_call_stack_critical_ptr, cpu)) {
+				scs_free_sdei();
+				return -ENOMEM;
+			}
+		} else {
+			per_cpu(sdei_shadow_call_stack_normal_ptr, cpu) =
+				per_cpu(sdei_shadow_call_stack_normal, cpu);
+			per_cpu(sdei_shadow_call_stack_critical_ptr, cpu) =
+				per_cpu(sdei_shadow_call_stack_critical, cpu);
+		}
 	}
+
+	return 0;
 }
arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.24.0.393.g34dc348eaf-goog
#267 lore raw

v7 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow, such as function graph tracing, although it may be possible to later change these features to modify the shadow stack instead. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (11):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI

 Makefile                             |   6 +
 arch/Kconfig                         |  35 ++++
 arch/arm64/Kconfig                   |   5 +
 arch/arm64/Makefile                  |   4 +
 arch/arm64/include/asm/scs.h         |  39 +++++
 arch/arm64/include/asm/suspend.h     |   2 +-
 arch/arm64/include/asm/thread_info.h |   3 +
 arch/arm64/kernel/Makefile           |   1 +
 arch/arm64/kernel/asm-offsets.c      |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S   |  11 +-
 arch/arm64/kernel/entry.S            |  46 ++++-
 arch/arm64/kernel/head.S             |   9 +
 arch/arm64/kernel/irq.c              |   2 +
 arch/arm64/kernel/process.c          |   2 +
 arch/arm64/kernel/scs.c              | 114 +++++++++++++
 arch/arm64/kernel/sdei.c             |   7 +
 arch/arm64/kernel/smp.c              |   4 +
 arch/arm64/kernel/vdso/Makefile      |   2 +-
 arch/arm64/kvm/hyp/Makefile          |   3 +
 arch/arm64/mm/proc.S                 |  14 ++
 drivers/base/node.c                  |   6 +
 fs/proc/meminfo.c                    |   4 +
 include/linux/compiler-clang.h       |   6 +
 include/linux/compiler_types.h       |   4 +
 include/linux/mmzone.h               |   3 +
 include/linux/scs.h                  |  57 +++++++
 init/init_task.c                     |   8 +
 kernel/Makefile                      |   1 +
 kernel/fork.c                        |   9 +
 kernel/sched/core.c                  |   2 +
 kernel/scs.c                         | 246 +++++++++++++++++++++++++++
 mm/page_alloc.c                      |   6 +
 mm/vmstat.c                          |   3 +
 33 files changed, 661 insertions(+), 6 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: b0be0eff1a5ab77d588b76bd8b1c92d5d17b3f73

Signature · 1 line
2.25.0.341.g760bfbb309-goog
#268 lore raw

PATCH v7 1/11 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 6a01b073915e..b2a1e5b704f4 100644
@@ -846,6 +846,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 48b5e103bdb0..1b16aa9a3fe5 100644
@@ -521,6 +521,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index f2cc0d118a0b..06231f936a7a 100644
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index ef82feb4bddc..2809e9f9f46b 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -454,6 +455,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -837,6 +840,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -896,6 +901,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 90e4b00ace89..a181c536e12e 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6038,6 +6039,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.25.0.341.g760bfbb309-goog
#269 lore raw

PATCH v7 2/11 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 98a31bafc8a2..874a8b428438 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 5334ad8fc7bd..1a379a0f2940 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index d047bf7d8fd4..284e428e71c8 100644
@@ -5340,6 +5340,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5362,6 +5365,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 78d53378db99..d0650391c8c1 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#270 lore raw

PATCH v7 3/11 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#271 lore raw

PATCH v7 4/11 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 1b16aa9a3fe5..0d746373c52e 100644
@@ -530,6 +530,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.25.0.341.g760bfbb309-goog
Reviewed-by on v7 4/11
#272 lore raw

Re: [PATCH v7 04/11] scs: disable when function graph tracing is enabled

Quoted 9 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

-Kees

Quoted 19 lines
---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 1b16aa9a3fe5..0d746373c52e 100644
@@ -530,6 +530,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.25.0.341.g760bfbb309-goog
Signature · 1 line
Kees Cook
#273 lore raw

PATCH v7 5/11 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index dca1a97751ab..ab26b448faa9 100644
@@ -65,6 +65,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#274 lore raw

PATCH v7 6/11 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index aafed6902411..7d37e3c70ff5 100644
@@ -56,6 +56,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -80,6 +82,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -96,6 +103,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#275 lore raw

PATCH v7 7/11 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.25.0.341.g760bfbb309-goog
maintainer Acked-by on v7 7/11
#276 lore raw

Re: [PATCH v7 07/11] arm64: efi: restore x18 if it was corrupted

Quoted 21 lines · Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +1 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:

arch/arm64/kernel/efi-rt-wrapper.S ENTRY(__efi_rt_asm_wrapper) · line 47

Quoted 10 lines
arch/arm64/kernel/efi-rt-wrapper.S +9 -0
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)

Will

#277 lore raw

PATCH v7 8/11 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#278 lore raw

PATCH v7 9/11 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..5843adef9ef6 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
Signature · 1 line
2.25.0.341.g760bfbb309-goog
maintainer Acked-by on v7 9/11
#279 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 16 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

---
 arch/arm64/kvm/hyp/Makefile | 3 +++
 1 file changed, 3 insertions(+)
arch/arm64/kvm/hyp/Makefile +0 -0
index ea710f674cb6..5843adef9ef6 100644

arch/arm64/kvm/hyp/Makefile GCOV_PROFILE := n · line 33

Quoted 6 lines
arch/arm64/kvm/hyp/Makefile +3 -0
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))

Will

#280 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Hi Sami,

Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

As you have an __attribute__((no_sanitize("shadow-call-stack"))), could we add that to __hyp_text instead? (its a smaller hammer!) All of KVM's EL2 code is marked __hyp_text, but that isn't everything in these files. Doing it like this would leave KVM's VHE-only paths covered.

As an example, with VHE the kernel and KVM both run at EL2, and KVM behaves differently:
kvm_vcpu_put_sysregs() in kvm/hyp/sysreg-sr.c is called from a preempt notifier as the EL2 registers are always accessible.

Thanks,

James

Quoted 11 lines
arch/arm64/kvm/hyp/Makefile +3 -0
index ea710f674cb6..5843adef9ef6 100644
@@ -28,3 +28,6 @@ GCOV_PROFILE	:= n
 KASAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
+
+# remove the SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
maintainer
#281 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 8 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

Quoted 8 lines

As you have an __attribute__((no_sanitize("shadow-call-stack"))), could we add that to __hyp_text instead? (its a smaller hammer!) All of KVM's EL2 code is marked __hyp_text, but that isn't everything in these files. Doing it like this would leave KVM's VHE-only paths covered.

As an example, with VHE the kernel and KVM both run at EL2, and KVM behaves differently:
kvm_vcpu_put_sysregs() in kvm/hyp/sysreg-sr.c is called from a preempt notifier as the EL2 registers are always accessible.

That's a good point, and I agree that it would be nice to have SCS covering the VHE paths. If you do that as a function attribute (which feels pretty fragile to me), then I guess we'll have to keep the -ffixed-x18 for the non-VHE code after all because GCC at least doesn't like having the register saving ABI specified on a per-function basis.

Will

reviewer
#282 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 5 lines · Will Deacon
Quoted 4 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Quoted 8 lines
Quoted 4 lines

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

When EL1 code calls up to hyp, it expects x18 to be preserved across the call, so hyp needs to either preserve it explicitly across a transitions from/to EL1 or always preserve it.

The latter is easiest since any code used by VHE hyp code will need x18 saved anyway (ans so any common hyp code needs to).

Thanks,
Mark.

maintainer
#283 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 11 lines · Mark Rutland
Quoted 10 lines · Will Deacon
Quoted 8 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

Quoted 7 lines
Quoted 3 lines

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

When EL1 code calls up to hyp, it expects x18 to be preserved across the call, so hyp needs to either preserve it explicitly across a transitions from/to EL1 or always preserve it.

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is that not sufficient?

Quoted 2 lines

The latter is easiest since any code used by VHE hyp code will need x18 saved anyway (ans so any common hyp code needs to).

I would personally prefer to split the VHE and non-VHE code so they can be compiled with separate options.

Will

reviewer
#284 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 16 lines · Will Deacon
Quoted 15 lines · Mark Rutland
Quoted 13 lines · Will Deacon
Quoted 8 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

Quoted 7 lines
Quoted 3 lines

When EL1 code calls up to hyp, it expects x18 to be preserved across the call, so hyp needs to either preserve it explicitly across a transitions from/to EL1 or always preserve it.

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is that not sufficient?

That covers the hyp->guest->hyp round trip, but not the host->hyp->host portion surrounding that.

Anywhere we use __call_hyp() expects x18 to be preserved across the call, and that's not only used to enter the guest. If we don't want to do that naturally at EL2, we'd probably have to add that to do_el2_call in arch/arm64/kvm/hyp/hyp-entry.S.

Thanks,
Mark.

maintainer
#285 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 21 lines · Mark Rutland
Quoted 20 lines · Will Deacon
Quoted 18 lines · Mark Rutland
Quoted 5 lines · Will Deacon
Quoted 4 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Quoted 8 lines
Quoted 4 lines

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

When EL1 code calls up to hyp, it expects x18 to be preserved across the call, so hyp needs to either preserve it explicitly across a transitions from/to EL1 or always preserve it.

Quoted 6 lines
Quoted 3 lines

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is that not sufficient?

That covers the hyp->guest->hyp round trip, but not the host->hyp->host portion surrounding that.

Thanks, I missed that. It's annoying that we'll end up needing /both/ -ffixed-x18 *and* the save/restore around guest transitions, but if we actually want to use SCS for the VHE code then I see that it will be required.

Sami -- can you restore -ffixed-x18 and then try the function attribute as suggested by James, please?

Will

#286 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 7 lines · Will Deacon

Thanks, I missed that. It's annoying that we'll end up needing /both/ -ffixed-x18 *and* the save/restore around guest transitions, but if we actually want to use SCS for the VHE code then I see that it will be required.

Sami -- can you restore -ffixed-x18 and then try the function attribute as suggested by James, please?

Sure. Adding __noscs to __hyp_text and not filtering out any of the flags in the Makefile appears to work. I'll update this in the next version.

Sami

maintainer
#287 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

In reply to Will Deacon
Quoted 24 lines · Will Deacon
Quoted 11 lines · Mark Rutland
Quoted 10 lines · Will Deacon
Quoted 8 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

Quoted 7 lines
Quoted 3 lines

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

When EL1 code calls up to hyp, it expects x18 to be preserved across the call, so hyp needs to either preserve it explicitly across a transitions from/to EL1 or always preserve it.

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is that not sufficient?

Quoted 5 lines
Quoted 2 lines

The latter is easiest since any code used by VHE hyp code will need x18 saved anyway (ans so any common hyp code needs to).

I would personally prefer to split the VHE and non-VHE code so they can be compiled with separate options.

This is going to generate a lot of code duplication (or at least object duplication), as the two code paths are intricately linked and splitting them to support different compilation options and/or calling conventions.

I'm not fundamentally opposed to that, but it should come with ways to still manage it as a unified code base as much as possible, as ways to discard the unused part at runtime (which should become easy to do once we have two distinct sets of objects).

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
maintainer
#288 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 42 lines · Marc Zyngier
Quoted 27 lines · Will Deacon
Quoted 15 lines · Mark Rutland
Quoted 5 lines · Will Deacon
Quoted 4 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Quoted 8 lines
Quoted 4 lines

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

Quoted 5 lines

When EL1 code calls up to hyp, it expects x18 to be preserved across the
call, so hyp needs to either preserve it explicitly across a transitions
from/to EL1 or always preserve it.

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is
that not sufficient?

Quoted 7 lines
Quoted 3 lines

The latter is easiest since any code used by VHE hyp code will need x18
saved anyway (ans so any common hyp code needs to).

I would personally prefer to split the VHE and non-VHE code so they can be
compiled with separate options.

This is going to generate a lot of code duplication (or at least object duplication),
as the two code paths are intricately linked and splitting them to support different
compilation options and/or calling conventions.

Quoted 4 lines

I'm not fundamentally opposed to that, but it should come with ways to still manage it as a unified code base as much as possible, as ways to discard the unused part at runtime (which should become easy to do once we have two distinct sets of objects).

Agreed, and I don't want to hold up the SCS patches because of this. I'm just nervous about the function attribute because I've only ever had terrible experiences with them. Maybe it will work this time (!)

Will

maintainer
#289 lore raw

Re: [PATCH v7 09/11] arm64: disable SCS for hypervisor code

Quoted 43 lines · Will Deacon
Quoted 42 lines · Marc Zyngier
Quoted 27 lines · Will Deacon
Quoted 15 lines · Mark Rutland
Quoted 5 lines · Will Deacon
Quoted 4 lines · James Morse
Quoted 2 lines · Sami Tolvanen

Filter out CC_FLAGS_SCS and -ffixed-x18 for code that runs at a different exception level.

Quoted 8 lines
Quoted 4 lines

Hmmm, there are two things being disabled here.

Stashing the lr in memory pointed to by VA won't work transparently at EL2 ... but shouldn't KVM's C code still treat x18 as a fixed register?

My review of v6 suggested dropping the -ffixed-x18 as well, since it's only introduced by SCS (in patch 5) and so isn't required by anything else. Why do you think it's needed?

Quoted 5 lines

When EL1 code calls up to hyp, it expects x18 to be preserved across the
call, so hyp needs to either preserve it explicitly across a transitions
from/to EL1 or always preserve it.

I thought we explicitly saved/restored it across the call after af12376814a5 ("arm64: kvm: stop treating register x18 as caller save"). Is
that not sufficient?

Quoted 7 lines
Quoted 3 lines

The latter is easiest since any code used by VHE hyp code will need x18
saved anyway (ans so any common hyp code needs to).

I would personally prefer to split the VHE and non-VHE code so they can be
compiled with separate options.

This is going to generate a lot of code duplication (or at least object duplication),
as the two code paths are intricately linked and splitting them to support different
compilation options and/or calling conventions.

Quoted 8 lines
Quoted 4 lines

I'm not fundamentally opposed to that, but it should come with ways to still manage it as a unified code base as much as possible, as ways to discard the unused part at runtime (which should become easy to do once we have two distinct sets of objects).

Agreed, and I don't want to hold up the SCS patches because of this. I'm just nervous about the function attribute because I've only ever had terrible experiences with them. Maybe it will work this time (!)

I have the same experience chasing missing __hyp_text attributes. Until we have tooling that picks on this *at compile time*, we'll have to play wack-a-mole with them...

        M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#290 lore raw

PATCH v7 10/11 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 32 ++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 5616cab8dfda..147aa56dea02 100644
@@ -65,6 +65,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1020,6 +1021,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index fc6488660f64..08fafc4da2cf 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index a5bdce8af65b..d485dc5cd196 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +30 -2
index 1b6b7a86625c..f9370d768494 100644
@@ -177,6 +177,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -400,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -895,6 +918,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..ca561de903d4 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index bbb0f0c145f6..df45eb09845b 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -507,6 +508,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index d4ed9a19d8fe..f2cb344f998c 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -358,6 +359,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.25.0.341.g760bfbb309-goog
#291 lore raw

PATCH v7 11/11 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index f9370d768494..42183895fb84 100644
@@ -1050,13 +1050,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1066,6 +1069,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.25.0.341.g760bfbb309-goog
Reviewed-by Tested-by on v7 11/11
#292 lore raw

Re: [PATCH v7 11/11] arm64: scs: add shadow stacks for SDEI

Hi Sami,

Quoted 3 lines · Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

Quoted 4 lines
arch/arm64/kernel/scs.c +0 -0
index eaadf5430baa..dddb7c56518b 100644

arch/arm64/kernel/scs.c line 46

Quoted 10 lines
arch/arm64/kernel/scs.c +9 -0
 
+static int scs_alloc_percpu(unsigned long * __percpu *ptr, int cpu)
+{
+	unsigned long *p;
+
+	p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+				 VMALLOC_START, VMALLOC_END,
+				 GFP_SCS, PAGE_KERNEL,
+				 0, cpu_to_node(cpu),
+				 __builtin_return_address(0));

(What makes this arch specific? arm64 has its own calls like this for the regular vmap stacks because it plays tricks with the alignment. Here the alignment requirement comes from the core SCS code... Would another architecture implement these scs_alloc_percpu()/scs_free_percpu() differently?)

Thanks,

James

#293 lore raw

Re: [PATCH v7 11/11] arm64: scs: add shadow stacks for SDEI

Quoted 10 lines · James Morse

Hi Sami,

Quoted 3 lines · Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

Thank you for taking the time to test this, James!

Quoted 5 lines
Quoted 4 lines
arch/arm64/kernel/scs.c +0 -0
index eaadf5430baa..dddb7c56518b 100644
Quoted 14 lines

arch/arm64/kernel/scs.c line 46

Quoted 9 lines
arch/arm64/kernel/scs.c +9 -0
+static int scs_alloc_percpu(unsigned long * __percpu *ptr, int cpu)
+{
+     unsigned long *p;
+
+     p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+                              VMALLOC_START, VMALLOC_END,
+                              GFP_SCS, PAGE_KERNEL,
+                              0, cpu_to_node(cpu),
+                              __builtin_return_address(0));

(What makes this arch specific? arm64 has its own calls like this for the regular vmap stacks because it plays tricks with the alignment. Here the alignment requirement comes from the core SCS code... Would another architecture implement these scs_alloc_percpu()/scs_free_percpu() differently?)

You are correct, these aren't necessarily specific to arm64. However, right now, we are not allocating per-CPU shadow stacks anywhere else, so this was a natural place for the helper functions. Would you prefer me to move these to kernel/scs.c instead?

Sami

#294 lore raw

Re: [PATCH v7 11/11] arm64: scs: add shadow stacks for SDEI

Hi Sami,

Quoted 29 lines · Sami Tolvanen
Quoted 13 lines · James Morse
Quoted 3 lines · Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

Quoted 4 lines
arch/arm64/kernel/scs.c +0 -0
index eaadf5430baa..dddb7c56518b 100644
Quoted 14 lines

arch/arm64/kernel/scs.c line 46

Quoted 9 lines
arch/arm64/kernel/scs.c +9 -0
+static int scs_alloc_percpu(unsigned long * __percpu *ptr, int cpu)
+{
+     unsigned long *p;
+
+     p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+                              VMALLOC_START, VMALLOC_END,
+                              GFP_SCS, PAGE_KERNEL,
+                              0, cpu_to_node(cpu),
+                              __builtin_return_address(0));

(What makes this arch specific? arm64 has its own calls like this for the regular vmap stacks because it plays tricks with the alignment. Here the alignment requirement comes from the core SCS code... Would another architecture implement these scs_alloc_percpu()/scs_free_percpu() differently?)

Quoted 3 lines

You are correct, these aren't necessarily specific to arm64. However, right now, we are not allocating per-CPU shadow stacks anywhere else, so this was a natural place for the helper functions.

Fair enough,

Quoted 1 line

Would you prefer me to move these to kernel/scs.c instead?

I have no preference, as long as they don't get duplicated later!

Thanks,

James

#295 lore raw
In reply to Sami Tolvanen

Hi Sami,

Quoted 16 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

Quoted 3 lines

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

I found I had to add:
| KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))

to drivers/firmware/efi/libstub/Makefile, to get this going.
I don't think there is much point supporting SCS for the EFIstub, its already isolated from the rest of the kernel's C code by the __efistub symbol prefix machinery, and trying to use it would expose us to buggy firmware at a point we can't handle it!

I can send a patch if its easier for you,

Thanks,

James

#296 lore raw
Quoted 4 lines · James Morse

I found I had to add:
| KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))

to drivers/firmware/efi/libstub/Makefile, to get this going.

Ah, good catch!

Quoted 3 lines

I don't think there is much point supporting SCS for the EFIstub, its already isolated from the rest of the kernel's C code by the __efistub symbol prefix machinery, and trying to use it would expose us to buggy firmware at a point we can't handle it!

Yes, fully agreed.

Quoted 1 line

I can send a patch if its easier for you,

It's not a problem, I will include a patch for this in v8.

Sami

#297 lore raw

v8 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow, such as function graph tracing, although it may be possible to later change these features to modify the shadow stack instead. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 +
 arch/Kconfig                          |  35 ++++
 arch/arm64/Kconfig                    |   5 +
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  39 ++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |   3 +
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 +-
 arch/arm64/kernel/entry.S             |  46 ++++-
 arch/arm64/kernel/head.S              |   9 +
 arch/arm64/kernel/irq.c               |   2 +
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               | 114 ++++++++++++
 arch/arm64/kernel/sdei.c              |   7 +
 arch/arm64/kernel/smp.c               |   4 +
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 ++
 drivers/base/node.c                   |   6 +
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 +
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  57 ++++++
 init/init_task.c                      |   8 +
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 +
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 246 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 +
 mm/vmstat.c                           |   3 +
 34 files changed, 662 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 2b72104b8c12504176fb5fc1442d6e54e31e338b

Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#298 lore raw

PATCH v8 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index aab38cb02b24..69119440b843 100644
@@ -845,6 +845,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 98de654b79b3..66b34fd0df54 100644
@@ -526,6 +526,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 60a1295f4384..2bc73d654593 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -454,6 +455,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -837,6 +840,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -896,6 +901,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 1a9983da4408..7473cd685560 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6036,6 +6037,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#299 lore raw

Re: [PATCH v8 01/12] add support for Clang's Shadow Call Stack (SCS)

Hi Sami,

a couple of minor tweaks:

Quoted 12 lines · Sami Tolvanen
arch/Kconfig +4 -0
index 98de654b79b3..66b34fd0df54 100644
@@ -526,6 +526,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow

arch/Kconfig config STACKPROTECTOR_STRONG · line 542

Quoted 10 lines
arch/Kconfig +10 -0
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found from

found in

arch/Kconfig config STACKPROTECTOR_STRONG · line 550

Quoted 8 lines
arch/Kconfig +8 -0
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable reading and writing arbitrary memory may

capable of

Quoted 5 lines
arch/Kconfig +5 -0
+	  be able to locate them and hijack control flow by modifying shadow
+	  stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"

arch/Kconfig config STACKPROTECTOR_STRONG · line 565

Quoted 10 lines
arch/Kconfig +7 -0
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help

thanks.

Signature · 1 line
~Randy
#300 lore raw

Re: [PATCH v8 01/12] add support for Clang's Shadow Call Stack (SCS)

Quoted 31 lines · Randy Dunlap

Hi Sami,

a couple of minor tweaks:

Quoted 12 lines · Sami Tolvanen
arch/Kconfig +4 -0
index 98de654b79b3..66b34fd0df54 100644
@@ -526,6 +526,40 @@ config STACKPROTECTOR_STRONG
        about 20% of all kernel functions, which increases the kernel code
        size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+     bool
+     help
+       An architecture should select this if it supports Clang's Shadow

arch/Kconfig config STACKPROTECTOR_STRONG · line 542

Quoted 10 lines
arch/Kconfig +10 -0
+       Call Stack, has asm/scs.h, and implements runtime support for shadow
+       stack switching.
+
+config SHADOW_CALL_STACK
+     bool "Clang Shadow Call Stack"
+     depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+     help
+       This option enables Clang's Shadow Call Stack, which uses a
+       shadow stack to protect function return addresses from being
+       overwritten by an attacker. More information can be found from

found in

Quoted 10 lines

arch/Kconfig config STACKPROTECTOR_STRONG · line 550

Quoted 8 lines
arch/Kconfig +8 -0
+       Clang's documentation:
+
+         https://clang.llvm.org/docs/ShadowCallStack.html
+
+       Note that security guarantees in the kernel differ from the ones
+       documented for user space. The kernel must store addresses of shadow
+       stacks used by other tasks and interrupt handlers in memory, which
+       means an attacker capable reading and writing arbitrary memory may

capable of

Thanks, Randy! I'll fix these in the next version.

Sami

#301 lore raw

PATCH v8 2/12 scs: add accounting

In reply to Sami Tolvanen

This change adds accounting for the memory allocated for shadow stacks.

---
 drivers/base/node.c    |  6 ++++++
 fs/proc/meminfo.c      |  4 ++++
 include/linux/mmzone.h |  3 +++
 kernel/scs.c           | 20 ++++++++++++++++++++
 mm/page_alloc.c        |  6 ++++++
 mm/vmstat.c            |  3 +++
 6 files changed, 42 insertions(+)
drivers/base/node.c +6 -0
index 98a31bafc8a2..874a8b428438 100644
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d AnonPages:      %8lu kB\n"
 		       "Node %d Shmem:          %8lu kB\n"
 		       "Node %d KernelStack:    %8lu kB\n"
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       "Node %d ShadowCallStack:%8lu kB\n"
+#endif
 		       "Node %d PageTables:     %8lu kB\n"
 		       "Node %d NFS_Unstable:   %8lu kB\n"
 		       "Node %d Bounce:         %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
 		       nid, K(i.sharedram),
 		       nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+		       nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 		       nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
 		       nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
 		       nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
fs/proc/meminfo.c +4 -0
index 8c1f1bb1a5ce..49768005a79e 100644
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "SUnreclaim:     ", sunreclaim);
 	seq_printf(m, "KernelStack:    %8lu kB\n",
 		   global_zone_page_state(NR_KERNEL_STACK_KB));
+#ifdef CONFIG_SHADOW_CALL_STACK
+	seq_printf(m, "ShadowCallStack:%8lu kB\n",
+		   global_zone_page_state(NR_KERNEL_SCS_BYTES) / 1024);
+#endif
 	show_val_kb(m, "PageTables:     ",
 		    global_zone_page_state(NR_PAGETABLE));
 
include/linux/mmzone.h +3 -0
index 462f6873905a..0a6f395abc68 100644
@@ -200,6 +200,9 @@ enum zone_stat_item {
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_PAGETABLE,		/* used for pagetables */
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	NR_KERNEL_SCS_BYTES,	/* measured in bytes */
+#endif
 	/* Second 128 byte cacheline */
 	NR_BOUNCE,
 #if IS_ENABLED(CONFIG_ZSMALLOC)
kernel/scs.c +20 -0
index 28abed21950c..5245e992c692 100644
@@ -12,6 +12,7 @@
 #include <linux/scs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <linux/vmstat.h>
 #include <asm/scs.h>
 
 static inline void *__scs_base(struct task_struct *tsk)
@@ -89,6 +90,11 @@ static void scs_free(void *s)
 	vfree_atomic(s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return vmalloc_to_page(__scs_base(tsk));
+}
+
 static int scs_cleanup(unsigned int cpu)
 {
 	int i;
@@ -135,6 +141,11 @@ static inline void scs_free(void *s)
 	kmem_cache_free(scs_cache, s);
 }
 
+static struct page *__scs_page(struct task_struct *tsk)
+{
+	return virt_to_page(__scs_base(tsk));
+}
+
 void __init scs_init(void)
 {
 	scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, SCS_SIZE,
@@ -153,6 +164,12 @@ void scs_task_reset(struct task_struct *tsk)
 	task_set_scs(tsk, __scs_base(tsk));
 }
 
+static void scs_account(struct task_struct *tsk, int account)
+{
+	mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_BYTES,
+		account * SCS_SIZE);
+}
+
 int scs_prepare(struct task_struct *tsk, int node)
 {
 	void *s;
@@ -162,6 +179,8 @@ int scs_prepare(struct task_struct *tsk, int node)
 		return -ENOMEM;
 
 	task_set_scs(tsk, s);
+	scs_account(tsk, 1);
+
 	return 0;
 }
 
@@ -182,6 +201,7 @@ void scs_release(struct task_struct *tsk)
 
 	WARN_ON(scs_corrupted(tsk));
 
+	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
 	scs_free(s);
 }
mm/page_alloc.c +6 -0
index 3c4eb750a199..1381b9d84e4c 100644
@@ -5340,6 +5340,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" managed:%lukB"
 			" mlocked:%lukB"
 			" kernel_stack:%lukB"
+#ifdef CONFIG_SHADOW_CALL_STACK
+			" shadow_call_stack:%lukB"
+#endif
 			" pagetables:%lukB"
 			" bounce:%lukB"
 			" free_pcp:%lukB"
@@ -5362,6 +5365,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(zone_managed_pages(zone)),
 			K(zone_page_state(zone, NR_MLOCK)),
 			zone_page_state(zone, NR_KERNEL_STACK_KB),
+#ifdef CONFIG_SHADOW_CALL_STACK
+			zone_page_state(zone, NR_KERNEL_SCS_BYTES) / 1024,
+#endif
 			K(zone_page_state(zone, NR_PAGETABLE)),
 			K(zone_page_state(zone, NR_BOUNCE)),
 			K(free_pcp),
mm/vmstat.c +3 -0
index 78d53378db99..d0650391c8c1 100644
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = {
 	"nr_mlock",
 	"nr_page_table_pages",
 	"nr_kernel_stack",
+#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
+	"nr_shadow_call_stack_bytes",
+#endif
 	"nr_bounce",
 #if IS_ENABLED(CONFIG_ZSMALLOC)
 	"nr_zspages",
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#302 lore raw

PATCH v8 3/12 scs: add support for stack usage debugging

In reply to Sami Tolvanen

Implements CONFIG_DEBUG_STACK_USAGE for shadow stacks. When enabled, also prints out the highest shadow stack usage per process.

---
 kernel/scs.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
kernel/scs.c +39 -0
index 5245e992c692..ad74d13f2c0f 100644
@@ -184,6 +184,44 @@ int scs_prepare(struct task_struct *tsk, int node)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_STACK_USAGE
+static inline unsigned long scs_used(struct task_struct *tsk)
+{
+	unsigned long *p = __scs_base(tsk);
+	unsigned long *end = scs_magic(p);
+	unsigned long s = (unsigned long)p;
+
+	while (p < end && READ_ONCE_NOCHECK(*p))
+		p++;
+
+	return (unsigned long)p - s;
+}
+
+static void scs_check_usage(struct task_struct *tsk)
+{
+	static DEFINE_SPINLOCK(lock);
+	static unsigned long highest;
+	unsigned long used = scs_used(tsk);
+
+	if (used <= highest)
+		return;
+
+	spin_lock(&lock);
+
+	if (used > highest) {
+		pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
+			tsk->comm, task_pid_nr(tsk), used);
+		highest = used;
+	}
+
+	spin_unlock(&lock);
+}
+#else
+static inline void scs_check_usage(struct task_struct *tsk)
+{
+}
+#endif
+
 bool scs_corrupted(struct task_struct *tsk)
 {
 	unsigned long *magic = scs_magic(__scs_base(tsk));
@@ -200,6 +238,7 @@ void scs_release(struct task_struct *tsk)
 		return;
 
 	WARN_ON(scs_corrupted(tsk));
+	scs_check_usage(tsk);
 
 	scs_account(tsk, -1);
 	task_set_scs(tsk, NULL);
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#303 lore raw

PATCH v8 4/12 scs: disable when function graph tracing is enabled

In reply to Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +1 -0
index 66b34fd0df54..4102b8e0eea9 100644
@@ -535,6 +535,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on !FUNCTION_GRAPH_TRACER
 	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
 	help
 	  This option enables Clang's Shadow Call Stack, which uses a
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
reviewer Reviewed-by on v8 4/12
#304 lore raw

Re: [PATCH v8 04/12] scs: disable when function graph tracing is enabled

Quoted 19 lines · Sami Tolvanen

The graph tracer hooks returns by modifying frame records on the (regular) stack, but with SCS the return address is taken from the shadow stack, and the value in the frame record has no effect. As we don't currently have a mechanism to determine the corresponding slot on the shadow stack (and to pass this through the ftrace infrastructure), for now let's disable SCS when the graph tracer is enabled.

---
 arch/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/Kconfig +0 -0
index 66b34fd0df54..4102b8e0eea9 100644

arch/Kconfig config ARCH_SUPPORTS_SHADOW_CALL_STACK · line 538

Quoted 4 lines
arch/Kconfig +1 -0
@@ -535,6 +535,7 @@ config ARCH_SUPPORTS_SHADOW_CALL_STACK
 
 config SHADOW_CALL_STACK
 	bool "Clang Shadow Call Stack"
+	depends on !FUNCTION_GRAPH_TRACER

Fangrui Song has implemented -fpatchable-function-entry in LLVM (for 10.x onwards), so we can support this when DYNAMIC_FTRACE_WITH_REGS is selected.

This can be:

	depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER

... and we can update the commit message to something like:

| With SCS the return address is taken from the shadow stack and the
| value in the frame record has no effect. The mcount based graph tracer
| hooks returns by modifying frame records on the (regular) stack, and
| thus is not compatible. The patchable-function-entry graph tracer
| used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved
| to the shadow stack, and is compatible.
|
| Modifying the mcount based graph tracer to work with SCS would require
| a mechanism to determine the corresponding slot on the shadow stack
| (and to pass this through the ftrace infrastructure), and we expect
| that everyone will eventually move to the patchable-function-entry
| based graph tracer anyway, so for now let's disable SCS when the
| mcount-based graph tracer is enabled.
|
| SCS and patchable-function-entry are both supported from LLVM 10.x.

Assuming you're happy with that:

Thanks,
Mark.

#305 lore raw

Re: [PATCH v8 04/12] scs: disable when function graph tracing is enabled

Quoted 26 lines · Mark Rutland

Fangrui Song has implemented -fpatchable-function-entry in LLVM (for 10.x onwards), so we can support this when DYNAMIC_FTRACE_WITH_REGS is selected.

This can be:

        depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER

... and we can update the commit message to something like:

| With SCS the return address is taken from the shadow stack and the
| value in the frame record has no effect. The mcount based graph tracer
| hooks returns by modifying frame records on the (regular) stack, and
| thus is not compatible. The patchable-function-entry graph tracer
| used for DYNAMIC_FTRACE_WITH_REGS modifies the LR before it is saved
| to the shadow stack, and is compatible.
|
| Modifying the mcount based graph tracer to work with SCS would require
| a mechanism to determine the corresponding slot on the shadow stack
| (and to pass this through the ftrace infrastructure), and we expect
| that everyone will eventually move to the patchable-function-entry
| based graph tracer anyway, so for now let's disable SCS when the
| mcount-based graph tracer is enabled.
|
| SCS and patchable-function-entry are both supported from LLVM 10.x.
Quoted 3 lines

Assuming you're happy with that:

Great, thanks for pointing that out! This looks good to me, I'll use this in v9.

Sami

#306 lore raw

PATCH v8 5/12 arm64: reserve x18 from general allocation with SCS

In reply to Sami Tolvanen

Reserve the x18 register from general allocation when SCS is enabled, because the compiler uses the register to store the current task's shadow stack pointer. Note that all external kernel modules must also be compiled with -ffixed-x18 if the kernel has SCS enabled.

---
 arch/arm64/Makefile | 4 ++++
 1 file changed, 4 insertions(+)
arch/arm64/Makefile +4 -0
index dca1a97751ab..ab26b448faa9 100644
@@ -65,6 +65,10 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
+KBUILD_CFLAGS	+= -ffixed-x18
+endif
+
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#307 lore raw

PATCH v8 6/12 arm64: preserve x18 when CPU is suspended

In reply to Sami Tolvanen

Don't lose the current task's shadow stack when the CPU is suspended.

---
 arch/arm64/include/asm/suspend.h |  2 +-
 arch/arm64/mm/proc.S             | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
arch/arm64/include/asm/suspend.h +1 -1
index 8939c87c4dce..0cde2f473971 100644
@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 12
+#define NR_CTX_REGS 13
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
arch/arm64/mm/proc.S +14 -0
index aafed6902411..7d37e3c70ff5 100644
@@ -56,6 +56,8 @@
  * cpu_do_suspend - save CPU registers context
  *
  * x0: virtual address of context pointer
+ *
+ * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
  */
 SYM_FUNC_START(cpu_do_suspend)
 	mrs	x2, tpidr_el0
@@ -80,6 +82,11 @@ alternative_endif
 	stp	x8, x9, [x0, #48]
 	stp	x10, x11, [x0, #64]
 	stp	x12, x13, [x0, #80]
+	/*
+	 * Save x18 as it may be used as a platform register, e.g. by shadow
+	 * call stack.
+	 */
+	str	x18, [x0, #96]
 	ret
 SYM_FUNC_END(cpu_do_suspend)
 
@@ -96,6 +103,13 @@ SYM_FUNC_START(cpu_do_resume)
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
 	ldp	x13, x14, [x0, #80]
+	/*
+	 * Restore x18, as it may be used as a platform register, and clear
+	 * the buffer to minimize the risk of exposure when used for shadow
+	 * call stack.
+	 */
+	ldr	x18, [x0, #96]
+	str	xzr, [x0, #96]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#308 lore raw

PATCH v8 7/12 arm64: efi: restore x18 if it was corrupted

In reply to Sami Tolvanen

If we detect a corrupted x18, restore the register before jumping back to potentially SCS instrumented code. This is safe, because the wrapper is called with preemption disabled and a separate shadow stack is used for interrupt handling.

---
 arch/arm64/kernel/efi-rt-wrapper.S | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/arm64/kernel/efi-rt-wrapper.S +10 -1
index 3fc71106cb2b..6ca6c0dc11a1 100644
@@ -34,5 +34,14 @@ ENTRY(__efi_rt_asm_wrapper)
 	ldp	x29, x30, [sp], #32
 	b.ne	0f
 	ret
-0:	b	efi_handle_corrupted_x18	// tail call
+0:
+	/*
+	 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
+	 * shadow stack pointer, which we need to restore before returning to
+	 * potentially instrumented code. This is safe because the wrapper is
+	 * called with preemption disabled and a separate shadow stack is used
+	 * for interrupts.
+	 */
+	mov	x18, x2
+	b	efi_handle_corrupted_x18	// tail call
 ENDPROC(__efi_rt_asm_wrapper)
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#309 lore raw

PATCH v8 8/12 arm64: vdso: disable Shadow Call Stack

In reply to Sami Tolvanen

Shadow stacks are only available in the kernel, so disable SCS instrumentation for the vDSO.

---
 arch/arm64/kernel/vdso/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/vdso/Makefile +1 -1
index dd2514bb1511..a87a4f11724e 100644
@@ -25,7 +25,7 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING
 
 VDSO_LDFLAGS := -Bsymbolic
 
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
 KBUILD_CFLAGS			+= $(DISABLE_LTO)
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#310 lore raw

PATCH v8 9/12 arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index a3a6a2ba9a63..0f0603f55ea0 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Reviewed-by on v8 9/12
#311 lore raw

Re: [PATCH v8 09/12] arm64: disable SCS for hypervisor code

Quoted 5 lines · Sami Tolvanen

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

-Kees

Quoted 20 lines
---
 arch/arm64/include/asm/kvm_hyp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/kvm_hyp.h +1 -1
index a3a6a2ba9a63..0f0603f55ea0 100644
@@ -13,7 +13,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
-#define __hyp_text __section(.hyp.text) notrace
+#define __hyp_text __section(.hyp.text) notrace __noscs
 
 #define read_sysreg_elx(r,nvh,vh)					\
 	({								\
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Signature · 1 line
Kees Cook
maintainer Acked-by on v8 9/12
#312 lore raw

Re: [PATCH v8 09/12] arm64: disable SCS for hypervisor code

In reply to Sami Tolvanen
Quoted 5 lines · On Tue, 18 Feb 2020 16:08:14 -0800 Sami Tolvanen <[email protected]>…

Disable SCS for code that runs at a different exception level by adding __noscs to __hyp_text.

	M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#313 lore raw

PATCH v8 10/12 arm64: implement Shadow Call Stack

In reply to Sami Tolvanen

This change implements shadow stack switching, initial SCS set-up, and interrupt shadow stacks for arm64.

---
 arch/arm64/Kconfig                   |  5 ++++
 arch/arm64/include/asm/scs.h         | 37 +++++++++++++++++++++++++
 arch/arm64/include/asm/thread_info.h |  3 +++
 arch/arm64/kernel/Makefile           |  1 +
 arch/arm64/kernel/asm-offsets.c      |  3 +++
 arch/arm64/kernel/entry.S            | 32 ++++++++++++++++++++--
 arch/arm64/kernel/head.S             |  9 +++++++
 arch/arm64/kernel/irq.c              |  2 ++
 arch/arm64/kernel/process.c          |  2 ++
 arch/arm64/kernel/scs.c              | 40 ++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c              |  4 +++
 11 files changed, 136 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
arch/arm64/Kconfig +5 -0
index 0b30e884e088..eae76686be77 100644
@@ -65,6 +65,7 @@ config ARM64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
+	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
 	select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1022,6 +1023,10 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 
+# Supported by clang >= 7.0
+config CC_HAVE_SHADOW_CALL_STACK
+	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
+
 config SECCOMP
 	bool "Enable seccomp to safely compute untrusted bytecode"
 	---help---
arch/arm64/include/asm/scs.h new file +37 -0
new file mode 100644
index 000000000000..c50d2b0c6c5f
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+static __always_inline void scs_save(struct task_struct *tsk)
+{
+	void *s;
+
+	asm volatile("mov %0, x18" : "=r" (s));
+	task_set_scs(tsk, s);
+}
+
+static inline void scs_overflow_check(struct task_struct *tsk)
+{
+	if (unlikely(scs_corrupted(tsk)))
+		panic("corrupted shadow stack detected inside scheduler\n");
+}
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+static inline void scs_init_irq(void) {}
+static inline void scs_save(struct task_struct *tsk) {}
+static inline void scs_overflow_check(struct task_struct *tsk) {}
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+
+#endif /* __ASSEMBLY __ */
+
+#endif /* _ASM_SCS_H */
arch/arm64/include/asm/thread_info.h +3 -0
index f0cec4160136..8c73764b9ed2 100644
@@ -41,6 +41,9 @@ struct thread_info {
 #endif
 		} preempt;
 	};
+#ifdef CONFIG_SHADOW_CALL_STACK
+	void			*shadow_call_stack;
+#endif
 };
 
 #define thread_saved_pc(tsk)	\
arch/arm64/kernel/Makefile +1 -0
index fc6488660f64..08fafc4da2cf 100644
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
+obj-$(CONFIG_SHADOW_CALL_STACK)		+= scs.o
 
 obj-y					+= vdso/ probes/
 obj-$(CONFIG_COMPAT_VDSO)		+= vdso32/
arch/arm64/kernel/asm-offsets.c +3 -0
index a5bdce8af65b..d485dc5cd196 100644
@@ -33,6 +33,9 @@ int main(void)
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
   DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+  DEFINE(TSK_TI_SCS,		offsetof(struct task_struct, thread_info.shadow_call_stack));
 #endif
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/entry.S +30 -2
index 9461d812ae27..4b18c3bbdea5 100644
@@ -177,6 +177,10 @@ alternative_cb_end
 
 	apply_ssbd 1, x22, x23
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
+	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
+#endif
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_current_task tsk
@@ -278,6 +282,12 @@ alternative_else_nop_endif
 	ct_user_enter
 	.endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	.if	\el == 0
+	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
+	.endif
+#endif
+
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	/*
 	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
@@ -383,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x24, x18		// preserve the original shadow stack
+#endif
 
 	/*
 	 * Compare sp with the base of the task stack.
@@ -400,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
 
 	/* switch to the irq stack */
 	mov	sp, x26
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* also switch to the irq shadow stack */
+	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
+#endif
+
 9998:
 	.endm
 
 	/*
-	 * x19 should be preserved between irq_stack_entry and
-	 * irq_stack_exit.
+	 * The callee-saved regs (x19-x29) should be preserved between
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
+	 * uses x20-x23 to store data for later use.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
+#ifdef CONFIG_SHADOW_CALL_STACK
+	mov	x18, x24
+#endif
 	.endm
 
 /* GPRs used by entry code */
@@ -895,6 +918,11 @@ ENTRY(cpu_switch_to)
 	ldr	lr, [x8]
 	mov	sp, x9
 	msr	sp_el0, x1
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str	x18, [x0, #TSK_TI_SCS]
+	ldr	x18, [x1, #TSK_TI_SCS]
+	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
arch/arm64/kernel/head.S +9 -0
index 989b1944cb71..ca561de903d4 100644
@@ -27,6 +27,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
+#include <asm/scs.h>
 #include <asm/smp.h>
 #include <asm/sysreg.h>
 #include <asm/thread_info.h>
@@ -424,6 +425,10 @@ __primary_switched:
 	stp	xzr, x30, [sp, #-16]!
 	mov	x29, sp
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	adr_l	x18, init_shadow_call_stack	// Set shadow call stack
+#endif
+
 	str_l	x21, __fdt_pointer, x5		// Save FDT pointer
 
 	ldr_l	x4, kimage_vaddr		// Save the offset between
@@ -731,6 +736,10 @@ __secondary_switched:
 	ldr	x2, [x0, #CPU_BOOT_TASK]
 	cbz	x2, __secondary_too_slow
 	msr	sp_el0, x2
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr	x18, [x2, #TSK_TI_SCS]		// set shadow call stack
+	str	xzr, [x2, #TSK_TI_SCS]		// limit visibility of saved SCS
+#endif
 	mov	x29, #0
 	mov	x30, #0
 	b	secondary_start_kernel
arch/arm64/kernel/irq.c +2 -0
index 04a327ccf84d..fe0ca522ff60 100644
@@ -21,6 +21,7 @@
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
+#include <asm/scs.h>
 
 unsigned long irq_err_count;
 
@@ -63,6 +64,7 @@ static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
arch/arm64/kernel/process.c +2 -0
index 00626057a384..9151616c354c 100644
@@ -52,6 +52,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/pointer_auth.h>
+#include <asm/scs.h>
 #include <asm/stacktrace.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
@@ -514,6 +515,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
 	uao_thread_switch(next);
 	ptrauth_thread_switch(next);
 	ssbs_thread_switch(next);
+	scs_overflow_check(next);
 
 	/*
 	 * Complete any pending TLB or cache maintenance on this CPU in case
arch/arm64/kernel/scs.c new file +40 -0
new file mode 100644
index 000000000000..eaadf5430baa
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Shadow Call Stack support.
+ *
+ * Copyright (C) 2019 Google LLC
+ */
+
+#include <linux/percpu.h>
+#include <linux/vmalloc.h>
+#include <asm/pgtable.h>
+#include <asm/scs.h>
+
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+#ifndef CONFIG_SHADOW_CALL_STACK_VMAP
+DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], irq_shadow_call_stack)
+	__aligned(SCS_SIZE);
+#endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SHADOW_CALL_STACK_VMAP
+		unsigned long *p;
+
+		p = __vmalloc_node_range(PAGE_SIZE, SCS_SIZE,
+					 VMALLOC_START, VMALLOC_END,
+					 GFP_SCS, PAGE_KERNEL,
+					 0, cpu_to_node(cpu),
+					 __builtin_return_address(0));
+
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = p;
+#else
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			per_cpu(irq_shadow_call_stack, cpu);
+#endif /* CONFIG_SHADOW_CALL_STACK_VMAP */
+	}
+}
arch/arm64/kernel/smp.c +4 -0
index d4ed9a19d8fe..f2cb344f998c 100644
@@ -46,6 +46,7 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/scs.h>
 #include <asm/smp_plat.h>
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -358,6 +359,9 @@ void cpu_die(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	/* Save the shadow stack pointer before exiting the idle task */
+	scs_save(current);
+
 	idle_task_exit();
 
 	local_daif_mask();
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#314 lore raw

PATCH v8 11/12 arm64: scs: add shadow stacks for SDEI

In reply to Sami Tolvanen

This change adds per-CPU shadow call stacks for the SDEI handler. Similarly to how the kernel stacks are handled, we add separate shadow stacks for normal and critical events.

---
 arch/arm64/include/asm/scs.h |   2 +
 arch/arm64/kernel/entry.S    |  14 ++++-
 arch/arm64/kernel/scs.c      | 106 +++++++++++++++++++++++++++++------
 arch/arm64/kernel/sdei.c     |   7 +++
 4 files changed, 112 insertions(+), 17 deletions(-)
arch/arm64/include/asm/scs.h +2 -0
index c50d2b0c6c5f..8e327e14bc15 100644
@@ -9,6 +9,7 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 
 extern void scs_init_irq(void);
+extern int scs_init_sdei(void);
 
 static __always_inline void scs_save(struct task_struct *tsk)
 {
@@ -27,6 +28,7 @@ static inline void scs_overflow_check(struct task_struct *tsk)
 #else /* CONFIG_SHADOW_CALL_STACK */
 
 static inline void scs_init_irq(void) {}
+static inline int scs_init_sdei(void) { return 0; }
 static inline void scs_save(struct task_struct *tsk) {}
 static inline void scs_overflow_check(struct task_struct *tsk) {}
 
arch/arm64/kernel/entry.S +13 -1
index 4b18c3bbdea5..2e2ce1b9ebf5 100644
@@ -1050,13 +1050,16 @@ ENTRY(__sdei_asm_handler)
 
 	mov	x19, x1
 
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+#endif
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * entry.S may have been using sp as a scratch register, find whether
 	 * this is a normal or critical event and switch to the appropriate
 	 * stack for this CPU.
 	 */
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
 	cbnz	w4, 1f
 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
 	b	2f
@@ -1066,6 +1069,15 @@ ENTRY(__sdei_asm_handler)
 	mov	sp, x5
 #endif
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+	/* Use a separate shadow call stack for normal and critical events */
+	cbnz	w4, 3f
+	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
+	b	4f
+3:	ldr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
+4:
+#endif
+
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
 	 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/scs.c +90 -16

121 lines not on this page · raw message

arch/arm64/kernel/sdei.c +7 -0
index d6259dac62b6..2854b9f7760a 100644
@@ -13,6 +13,7 @@
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -162,6 +163,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#315 lore raw

PATCH v8 12/12 efi/libstub: disable SCS

In reply to Sami Tolvanen

Disable SCS for the EFI stub and allow x18 to be used.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 98a81576213d..dff9fa5a3f1c 100644
@@ -30,6 +30,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Reviewed-by on v8 12/12
#316 lore raw

Re: [PATCH v8 12/12] efi/libstub: disable SCS

Quoted 4 lines · Sami Tolvanen

Disable SCS for the EFI stub and allow x18 to be used.

-Kees

Quoted 21 lines
---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +3 -0
index 98a81576213d..dff9fa5a3f1c 100644
@@ -30,6 +30,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -DDISABLE_BRANCH_PROFILING \
 				   $(call cc-option,-fno-stack-protector) \
 				   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+
 GCOV_PROFILE			:= n
 KASAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
Signature · 6 lines
Kees Cook

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
#317 lore raw

Re: [PATCH v8 12/12] efi/libstub: disable SCS

In reply to Sami Tolvanen
Quoted 14 lines · Sami Tolvanen

Disable SCS for the EFI stub and allow x18 to be used.

---
 drivers/firmware/efi/libstub/Makefile | 3 +++
 1 file changed, 3 insertions(+)
drivers/firmware/efi/libstub/Makefile +0 -0
index 98a81576213d..dff9fa5a3f1c 100644

drivers/firmware/efi/libstub/Makefile KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ · line 35

Quoted 6 lines
drivers/firmware/efi/libstub/Makefile +3 -0
@@ -30,6 +30,9 @@ KBUILD_CFLAGS                 := $(cflags-y) -DDISABLE_BRANCH_PROFILING \
                                   $(call cc-option,-fno-stack-protector) \
                                   -D__DISABLE_EXPORTS
 
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+

I don't see why you'd need to remove -ffixed-x18 again here. Not using x18 anywhere in the kernel is a much more maintainable approach.

In fact, now that I think of it, the EFI AArch64 platform binding forbids the use of x18, so it would be better to add the -ffixed-x18 unconditionally for arm64 (even though the reason it forbids it is to ensure compatibility with an OS using it as a platform register, and so nothing is actually broken atm).

Quoted 6 lines
drivers/firmware/efi/libstub/Makefile +0 -1
 GCOV_PROFILE                   := n
 KASAN_SANITIZE                 := n
 UBSAN_SANITIZE                 := n
--

2.25.0.265.gbab2e86ba0-goog

#318 lore raw

Re: [PATCH v8 12/12] efi/libstub: disable SCS

Quoted 9 lines · Ard Biesheuvel

drivers/firmware/efi/libstub/Makefile KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ · line 35

Quoted 4 lines · Sami Tolvanen
drivers/firmware/efi/libstub/Makefile +3 -0
+#  remove SCS flags from all objects in this directory
+KBUILD_CFLAGS := $(filter-out -ffixed-x18 $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+

I don't see why you'd need to remove -ffixed-x18 again here. Not using x18 anywhere in the kernel is a much more maintainable approach.

Sure, I will drop -ffixed-x18 from here in v9. Thanks,

Sami

#319 lore raw
In reply to Sami Tolvanen

Hi Sami,

(CC: +Marc)

Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites.

I took this for a spin on some real hardware. cpu-idle, kexec hibernate etc all work great... but starting a KVM guest causes the CPU to get stuck in EL2.

With CONFIG_SHADOW_CALL_STACK disabled, this doesn't happen ... so its something about the feature being enabled.

I'm using clang-9 from debian bullseye/sid. (I tried to build tip of tree ... that doesn't go so well on arm64)

KVM takes an instruction abort from EL2 to EL2, because some of the code it runs is not mapped at EL2:

| ffffa00011588308 <__kvm_tlb_flush_local_vmid>:
| ffffa00011588308:       d10103ff        sub     sp, sp, #0x40
| ffffa0001158830c:       f90013f3        str     x19, [sp, #32]
| ffffa00011588310:       a9037bfd        stp     x29, x30, [sp, #48]
| ffffa00011588314:       9100c3fd        add     x29, sp, #0x30
| ffffa00011588318:       97ae18bf        bl      ffffa0001010e614 <__kern_hyp_va>

INSTRUCTION ABORT!

| ffffa0001158831c:       f9400000        ldr     x0, [x0]
| ffffa00011588320:       97ae18bd        bl      ffffa0001010e614 <__kern_hyp_va>
| ffffa00011588324:       aa0003f3        mov     x19, x0
| ffffa00011588328:       97ae18c1        bl      ffffa0001010e62c <has_vhe>

__kern_hyp_va() is static-inline which is patched wherever it appears at boot with the EL2 ASLR values, it converts a kernel linear-map address to its EL2 KVM alias:

| ffffa0001010dc5c <__kern_hyp_va>:
| ffffa0001010dc5c:       92400000        and     x0, x0, #0x1
| ffffa0001010dc60:       93c00400        ror     x0, x0, #1
| ffffa0001010dc64:       91000000        add     x0, x0, #0x0
| ffffa0001010dc68:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa0001010dc6c:       93c0fc00        ror     x0, x0, #63
| ffffa0001010dc70:       d65f03c0        ret

The problem here is where __kern_hyp_va() is. Its outside the __hyp_text section:

| morse@eglon:~/kernel/linux-pigs$ nm -s vmlinux | grep hyp_text
| ffffa0001158b800 T __hyp_text_end
| ffffa000115838a0 T __hyp_text_start

If I disable CONFIG_SHADOW_CALL_STACK in Kconfig, I get:

| ffffa00011527fe0 <__kvm_tlb_flush_local_vmid>:
| ffffa00011527fe0:       d100c3ff        sub     sp, sp, #0x30
| ffffa00011527fe4:       a9027bfd        stp     x29, x30, [sp, #32]
| ffffa00011527fe8:       910083fd        add     x29, sp, #0x20
| ffffa00011527fec:       92400000        and     x0, x0, #0x1
| ffffa00011527ff0:       93c00400        ror     x0, x0, #1
| ffffa00011527ff4:       91000000        add     x0, x0, #0x0
| ffffa00011527ff8:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011527ffc:       93c0fc00        ror     x0, x0, #63
| ffffa00011528000:       f9400000        ldr     x0, [x0]
| ffffa00011528004:       910023e1        add     x1, sp, #0x8
| ffffa00011528008:       92400000        and     x0, x0, #0x1
| ffffa0001152800c:       93c00400        ror     x0, x0, #1
| ffffa00011528010:       91000000        add     x0, x0, #0x0
| ffffa00011528014:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011528018:       93c0fc00        ror     x0, x0, #63
| ffffa0001152801c:       97ffff78        bl      ffffa00011527dfc <__tlb_switch_>
| ffffa00011528020:       d508871f        tlbi    vmalle1
| ffffa00011528024:       d503201f        nop

This looks like reserving x18 is causing Clang to not-inline the __kern_hyp_va() calls, losing the vitally important section information. (I can see why the compiler thinks this is fair)

Is this a known, er, thing, with clang-9?

From eyeballing the disassembly __always_inline on __kern_hyp_va() is enough of a hint to stop this, ... with this configuration of clang-9. But KVM still doesn't work, so it isn't the only inlining decision KVM relies on that is changed by SCS.

I suspect repainting all KVM's 'inline' with __always_inline will fix it. (yuck!) I'll try tomorrow.

I don't think keeping the compiler-flags as they are today for KVM is the right thing to do, it could lead to x18 getting corrupted with the shared vhe/non-vhe code. Splitting that code up would lead to duplication.

(hopefully objtool will be able to catch these at build time)

Thanks,

James

Quoted 7 lines

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory

#320 lore raw
Quoted 82 lines · James Morse

Hi Sami,

(CC: +Marc)

Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites.

I took this for a spin on some real hardware. cpu-idle, kexec hibernate etc all work great... but starting a KVM guest causes the CPU to get stuck in EL2.

With CONFIG_SHADOW_CALL_STACK disabled, this doesn't happen ... so its something about the feature being enabled.

I'm using clang-9 from debian bullseye/sid. (I tried to build tip of tree ... that doesn't go so well on arm64)

KVM takes an instruction abort from EL2 to EL2, because some of the code it runs is not mapped at EL2:

| ffffa00011588308 <__kvm_tlb_flush_local_vmid>:
| ffffa00011588308:       d10103ff        sub     sp, sp, #0x40
| ffffa0001158830c:       f90013f3        str     x19, [sp, #32]
| ffffa00011588310:       a9037bfd        stp     x29, x30, [sp, #48]
| ffffa00011588314:       9100c3fd        add     x29, sp, #0x30
| ffffa00011588318:       97ae18bf        bl      ffffa0001010e614 <__kern_hyp_va>

INSTRUCTION ABORT!

| ffffa0001158831c:       f9400000        ldr     x0, [x0]
| ffffa00011588320:       97ae18bd        bl      ffffa0001010e614 <__kern_hyp_va>
| ffffa00011588324:       aa0003f3        mov     x19, x0
| ffffa00011588328:       97ae18c1        bl      ffffa0001010e62c <has_vhe>

__kern_hyp_va() is static-inline which is patched wherever it appears at boot with the EL2 ASLR values, it converts a kernel linear-map address to its EL2 KVM alias:

| ffffa0001010dc5c <__kern_hyp_va>:
| ffffa0001010dc5c:       92400000        and     x0, x0, #0x1
| ffffa0001010dc60:       93c00400        ror     x0, x0, #1
| ffffa0001010dc64:       91000000        add     x0, x0, #0x0
| ffffa0001010dc68:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa0001010dc6c:       93c0fc00        ror     x0, x0, #63
| ffffa0001010dc70:       d65f03c0        ret

The problem here is where __kern_hyp_va() is. Its outside the __hyp_text section:

| morse@eglon:~/kernel/linux-pigs$ nm -s vmlinux | grep hyp_text
| ffffa0001158b800 T __hyp_text_end
| ffffa000115838a0 T __hyp_text_start

If I disable CONFIG_SHADOW_CALL_STACK in Kconfig, I get:

| ffffa00011527fe0 <__kvm_tlb_flush_local_vmid>:
| ffffa00011527fe0:       d100c3ff        sub     sp, sp, #0x30
| ffffa00011527fe4:       a9027bfd        stp     x29, x30, [sp, #32]
| ffffa00011527fe8:       910083fd        add     x29, sp, #0x20
| ffffa00011527fec:       92400000        and     x0, x0, #0x1
| ffffa00011527ff0:       93c00400        ror     x0, x0, #1
| ffffa00011527ff4:       91000000        add     x0, x0, #0x0
| ffffa00011527ff8:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011527ffc:       93c0fc00        ror     x0, x0, #63
| ffffa00011528000:       f9400000        ldr     x0, [x0]
| ffffa00011528004:       910023e1        add     x1, sp, #0x8
| ffffa00011528008:       92400000        and     x0, x0, #0x1
| ffffa0001152800c:       93c00400        ror     x0, x0, #1
| ffffa00011528010:       91000000        add     x0, x0, #0x0
| ffffa00011528014:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011528018:       93c0fc00        ror     x0, x0, #63
| ffffa0001152801c:       97ffff78        bl      ffffa00011527dfc <__tlb_switch_>
| ffffa00011528020:       d508871f        tlbi    vmalle1
| ffffa00011528024:       d503201f        nop

This looks like reserving x18 is causing Clang to not-inline the __kern_hyp_va() calls, losing the vitally important section information. (I can see why the compiler thinks this is fair)

Quoted 9 lines

Is this a known, er, thing, with clang-9?

From eyeballing the disassembly __always_inline on __kern_hyp_va() is enough of a hint to stop this, ... with this configuration of clang-9. But KVM still doesn't work, so it isn't the only inlining decision KVM relies on that is changed by SCS.

I suspect repainting all KVM's 'inline' with __always_inline will fix it. (yuck!) I'll try tomorrow.

If we are relying on the inlining for correctness, these should have been __always_inline to begin with, and yuckness aside, I don't think there's anything wrong with that.

Quoted 6 lines

I don't think keeping the compiler-flags as they are today for KVM is the right thing to do, it could lead to x18 getting corrupted with the shared vhe/non-vhe code. Splitting that code up would lead to duplication.

(hopefully objtool will be able to catch these at build time)

I don't see why we should selectively en/disable the reservation of x18 (as I argued in the context of the EFI libstub patch as well). Just reserving it everywhere shouldn't hurt performance, and removes the need to prove that we reserved it in all the right places.

maintainer
#321 lore raw
Quoted 89 lines · Ard Biesheuvel
Quoted 88 lines · James Morse

Hi Sami,

(CC: +Marc)

Quoted 3 lines · Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites.

I took this for a spin on some real hardware. cpu-idle, kexec hibernate etc all work great... but starting a KVM guest causes the CPU to get stuck in EL2.

With CONFIG_SHADOW_CALL_STACK disabled, this doesn't happen ... so its something about the feature being enabled.

I'm using clang-9 from debian bullseye/sid. (I tried to build tip of tree ... that doesn't go so well on arm64)

KVM takes an instruction abort from EL2 to EL2, because some of the code it runs is not mapped at EL2:

| ffffa00011588308 <__kvm_tlb_flush_local_vmid>:
| ffffa00011588308:       d10103ff        sub     sp, sp, #0x40
| ffffa0001158830c:       f90013f3        str     x19, [sp, #32]
| ffffa00011588310:       a9037bfd        stp     x29, x30, [sp, #48]
| ffffa00011588314:       9100c3fd        add     x29, sp, #0x30
| ffffa00011588318:       97ae18bf        bl      ffffa0001010e614 <__kern_hyp_va>

INSTRUCTION ABORT!

| ffffa0001158831c:       f9400000        ldr     x0, [x0]
| ffffa00011588320:       97ae18bd        bl      ffffa0001010e614 <__kern_hyp_va>
| ffffa00011588324:       aa0003f3        mov     x19, x0
| ffffa00011588328:       97ae18c1        bl      ffffa0001010e62c <has_vhe>

__kern_hyp_va() is static-inline which is patched wherever it appears at boot with the EL2 ASLR values, it converts a kernel linear-map address to its EL2 KVM alias:

| ffffa0001010dc5c <__kern_hyp_va>:
| ffffa0001010dc5c:       92400000        and     x0, x0, #0x1
| ffffa0001010dc60:       93c00400        ror     x0, x0, #1
| ffffa0001010dc64:       91000000        add     x0, x0, #0x0
| ffffa0001010dc68:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa0001010dc6c:       93c0fc00        ror     x0, x0, #63
| ffffa0001010dc70:       d65f03c0        ret

The problem here is where __kern_hyp_va() is. Its outside the __hyp_text section:

| morse@eglon:~/kernel/linux-pigs$ nm -s vmlinux | grep hyp_text
| ffffa0001158b800 T __hyp_text_end
| ffffa000115838a0 T __hyp_text_start

If I disable CONFIG_SHADOW_CALL_STACK in Kconfig, I get:

| ffffa00011527fe0 <__kvm_tlb_flush_local_vmid>:
| ffffa00011527fe0:       d100c3ff        sub     sp, sp, #0x30
| ffffa00011527fe4:       a9027bfd        stp     x29, x30, [sp, #32]
| ffffa00011527fe8:       910083fd        add     x29, sp, #0x20
| ffffa00011527fec:       92400000        and     x0, x0, #0x1
| ffffa00011527ff0:       93c00400        ror     x0, x0, #1
| ffffa00011527ff4:       91000000        add     x0, x0, #0x0
| ffffa00011527ff8:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011527ffc:       93c0fc00        ror     x0, x0, #63
| ffffa00011528000:       f9400000        ldr     x0, [x0]
| ffffa00011528004:       910023e1        add     x1, sp, #0x8
| ffffa00011528008:       92400000        and     x0, x0, #0x1
| ffffa0001152800c:       93c00400        ror     x0, x0, #1
| ffffa00011528010:       91000000        add     x0, x0, #0x0
| ffffa00011528014:       91400000        add     x0, x0, #0x0, lsl #12
| ffffa00011528018:       93c0fc00        ror     x0, x0, #63
| ffffa0001152801c:       97ffff78        bl      ffffa00011527dfc <__tlb_switch_>
| ffffa00011528020:       d508871f        tlbi    vmalle1
| ffffa00011528024:       d503201f        nop

This looks like reserving x18 is causing Clang to not-inline the __kern_hyp_va() calls, losing the vitally important section information. (I can see why the compiler thinks this is fair)

Is this a known, er, thing, with clang-9?

From eyeballing the disassembly __always_inline on __kern_hyp_va() is enough of a hint to stop this, ... with this configuration of clang-9. But KVM still doesn't work, so it isn't the only inlining decision KVM relies on that is changed by SCS.

Quoted 7 lines
Quoted 3 lines

I suspect repainting all KVM's 'inline' with __always_inline will fix it. (yuck!) I'll try tomorrow.

If we are relying on the inlining for correctness, these should have been __always_inline to begin with, and yuckness aside, I don't think there's anything wrong with that.

Agreed. Not having __always_inline is definitely an oversight, and we should fix it ASAP (hell knows what another compiler could produce...). And the whole EL2 aliasing is utter yuck already, this isn't going to make things much worse...

I can queue something today for __kern_hyp_va(), but I'd like to make sure there isn't other silly mistakes like this one somewhere...

Quoted 11 lines
Quoted 6 lines

I don't think keeping the compiler-flags as they are today for KVM is the right thing to do, it could lead to x18 getting corrupted with the shared vhe/non-vhe code. Splitting that code up would lead to duplication.

(hopefully objtool will be able to catch these at build time)

I don't see why we should selectively en/disable the reservation of x18 (as I argued in the context of the EFI libstub patch as well). Just reserving it everywhere shouldn't hurt performance, and removes the need to prove that we reserved it in all the right places.

I'd certainly like to keep things simple if we can.

          M.
Signature · 1 line
Jazz is not dead. It just smells funny...
#322 lore raw
In reply to James Morse
Quoted 3 lines · James Morse

This looks like reserving x18 is causing Clang to not-inline the __kern_hyp_va() calls, losing the vitally important section information. (I can see why the compiler thinks this is fair)

Thanks for catching this. This doesn't appear to be caused by reserving x18, it looks like SCS itself is causing clang to avoid inlining these. If I add __noscs to __kern_hyp_va(), clang inlines the function again. __always_inline also works, as you pointed out.

Quoted 1 line

Is this a known, er, thing, with clang-9?

I can reproduce this with ToT clang as well.

Quoted 2 lines

I suspect repainting all KVM's 'inline' with __always_inline will fix it. (yuck!) I'll try tomorrow.

I think switching to __always_inline is the correct solution here.

Sami

#323 lore raw

v9 cover add support for Clang's Shadow Call Stack

In reply to Sami Tolvanen

This patch series adds support for Clang's Shadow Call Stack (SCS) mitigation, which uses a separately allocated shadow stack to protect against return address overwrites. More information can be found here:

https://clang.llvm.org/docs/ShadowCallStack.html

SCS provides better protection against traditional buffer overflows than CONFIG_STACKPROTECTOR_*, but it should be noted that SCS security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by inactive tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

SCS is currently supported only on arm64, where the compiler requires the x18 register to be reserved for holding the current task's shadow stack pointer.

With -fsanitize=shadow-call-stack, the compiler injects instructions to all non-leaf C functions to store the return address to the shadow stack, and unconditionally load it again before returning. As a result, SCS is currently incompatible with features that rely on modifying function return addresses in the kernel stack to alter control flow. A copy of the return address is still kept in the kernel stack for compatibility with stack unwinding, for example.

SCS has a minimal performance overhead, but allocating shadow stacks increases kernel memory usage. The feature is therefore mostly useful on hardware that lacks support for PAC instructions.

Changes in v9:
- Fixed grammar in the Kconfig help text.
- Changed Kconfig to allow SCS to be selected with the patchable- function-entry graph tracer.
- Changed the EFI stub patch to not filter out -ffixed-x18, only SCS flags.

Changes in v8:
- Added __noscs to __hyp_text instead of filtering SCS flags from the entire arch/arm64/kvm/hyp directory.
- Added a patch to filter out -ffixed-x18 and SCS flags from the EFI stub.

Changes in v7:
- Changed irq_stack_entry/exit to store the shadow stack pointer in x24 instead of x20 as kernel_entry uses x20-x23 to store data that can be used later. Updated the comment as well.
- Changed the Makefile in arch/arm64/kvm/hyp to also filter out -ffixed-x18.
- Changed SHADOW_CALL_STACK to depend on !FUNCTION_GRAPH_TRACER instead of not selecting HAVE_FUNCTION_GRAPH_TRACER with SCS.
- Removed ifdefs from the EFI wrapper and updated the comment to explain why we are restoring x18.
- Rebased as Ard's x18 patches that were part of this series have already been merged.

Changes in v6:
- Updated comment in the EFI RT wrapper to include the explanation from the commit message.
- Fixed the SHADOW_CALL_STACK_VMAP config option and the compilation errors in scs_init_irq()
- Updated the comment in entry.S to Mark's suggestion
- Fixed the WARN_ON in scs_init() to trip only when the return value for cpuhp_setup_state() is < 0.
- Removed ifdefs from the code in arch/arm64/kernel/scs.c and added separate shadow stacks for the SDEI handler

Changes in v5:
- Updated the comment in __scs_base() to Mark's suggestion
- Changed all instances of uintptr_t to unsigned long
- Added allocation poisoning for KASAN to catch unintentional shadow stack accesses; moved set_set_magic before poisoning and switched scs_used() and scs_corrupted() to access the buffer using READ_ONCE_NOCHECK() instead
- Changed scs_free() to check for NULL instead of zero
- Renamed SCS_CACHE_SIZE to NR_CACHED_SCS
- Added a warning if cpuhp_setup_state fails in scs_init()
- Dropped patches disabling kretprobes after confirming there's no functional conflict with SCS instrumentation
- Added an explanation to the commit message why function graph tracing and SCS are incompatible
- Removed the ifdefs from arch/arm64/mm/proc.S and added comments explaining why we are saving and restoring x18
- Updated scs_check_usage format to include process information

Changes in v4:
- Fixed authorship for Ard's patches
- Added missing commit messages
- Commented code that clears SCS from thread_info
- Added a comment about SCS_END_MAGIC being non-canonical

Changes in v3:
- Switched to filter-out for removing SCS flags in Makefiles
- Changed the __noscs attribute to use __no_sanitize__("...") instead of no_sanitize("...")
- Cleaned up inline function definitions and moved task_scs() into a macro
- Cleaned up scs_free() and scs_magic()
- Moved SCS initialization into dup_task_struct() and removed the now unused scs_task_init()
- Added comments to __scs_base() and scs_task_reset() to better document design choices
- Changed copy_page to make the offset and bias explicit

Changes in v2:
- Changed Ard's KVM patch to use x29 instead of x18 for the guest context, which makes restore_callee_saved_regs cleaner
- Updated help text (and commit messages) to point out differences in security properties compared to user space SCS
- Cleaned up config options: removed the ROP protection choice, replaced the CC_IS_CLANG dependency with an arch-specific cc-option test, and moved disabling of incompatible config options to an arch-specific Kconfig
- Added CC_FLAGS_SCS, which are filtered out where needed instead of using DISABLE_SCS
- Added a __has_feature guard around __noscs for older clang versions

Sami Tolvanen (12):
add support for Clang's Shadow Call Stack (SCS)
scs: add accounting
scs: add support for stack usage debugging
scs: disable when function graph tracing is enabled
arm64: reserve x18 from general allocation with SCS
arm64: preserve x18 when CPU is suspended
arm64: efi: restore x18 if it was corrupted
arm64: vdso: disable Shadow Call Stack
arm64: disable SCS for hypervisor code
arm64: implement Shadow Call Stack
arm64: scs: add shadow stacks for SDEI efi/libstub: disable SCS

 Makefile                              |   6 +
 arch/Kconfig                          |  35 ++++
 arch/arm64/Kconfig                    |   5 +
 arch/arm64/Makefile                   |   4 +
 arch/arm64/include/asm/kvm_hyp.h      |   2 +-
 arch/arm64/include/asm/scs.h          |  39 ++++
 arch/arm64/include/asm/suspend.h      |   2 +-
 arch/arm64/include/asm/thread_info.h  |   3 +
 arch/arm64/kernel/Makefile            |   1 +
 arch/arm64/kernel/asm-offsets.c       |   3 +
 arch/arm64/kernel/efi-rt-wrapper.S    |  11 +-
 arch/arm64/kernel/entry.S             |  46 ++++-
 arch/arm64/kernel/head.S              |   9 +
 arch/arm64/kernel/irq.c               |   2 +
 arch/arm64/kernel/process.c           |   2 +
 arch/arm64/kernel/scs.c               | 114 ++++++++++++
 arch/arm64/kernel/sdei.c              |   7 +
 arch/arm64/kernel/smp.c               |   4 +
 arch/arm64/kernel/vdso/Makefile       |   2 +-
 arch/arm64/mm/proc.S                  |  14 ++
 drivers/base/node.c                   |   6 +
 drivers/firmware/efi/libstub/Makefile |   3 +
 fs/proc/meminfo.c                     |   4 +
 include/linux/compiler-clang.h        |   6 +
 include/linux/compiler_types.h        |   4 +
 include/linux/mmzone.h                |   3 +
 include/linux/scs.h                   |  57 ++++++
 init/init_task.c                      |   8 +
 kernel/Makefile                       |   1 +
 kernel/fork.c                         |   9 +
 kernel/sched/core.c                   |   2 +
 kernel/scs.c                          | 246 ++++++++++++++++++++++++++
 mm/page_alloc.c                       |   6 +
 mm/vmstat.c                           |   3 +
 34 files changed, 662 insertions(+), 7 deletions(-)
 create mode 100644 arch/arm64/include/asm/scs.h
 create mode 100644 arch/arm64/kernel/scs.c
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c

base-commit: 63623fd44972d1ed2bfb6e0fb631dfcf547fd1e7

Signature · 1 line
2.25.0.265.gbab2e86ba0-goog
#324 lore raw

PATCH v9 1/12 add support for Clang's Shadow Call Stack (SCS)

This change adds generic support for Clang's Shadow Call Stack, which uses a shadow stack to protect return addresses from being overwritten by an attacker. Details are available here:

https://clang.llvm.org/docs/ShadowCallStack.html

Note that security guarantees in the kernel differ from the ones documented for user space. The kernel must store addresses of shadow stacks used by other tasks and interrupt handlers in memory, which means an attacker capable reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying shadow stacks that are not currently in use.

---
 Makefile                       |   6 ++
 arch/Kconfig                   |  34 ++++++
 include/linux/compiler-clang.h |   6 ++
 include/linux/compiler_types.h |   4 +
 include/linux/scs.h            |  57 ++++++++++
 init/init_task.c               |   8 ++
 kernel/Makefile                |   1 +
 kernel/fork.c                  |   9 ++
 kernel/sched/core.c            |   2 +
 kernel/scs.c                   | 187 +++++++++++++++++++++++++++++++++
 10 files changed, 314 insertions(+)
 create mode 100644 include/linux/scs.h
 create mode 100644 kernel/scs.c
Makefile +6 -0
index 0914049d2929..ea465905b399 100644
@@ -845,6 +845,12 @@ ifdef CONFIG_LIVEPATCH
 KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+CC_FLAGS_SCS	:= -fsanitize=shadow-call-stack
+KBUILD_CFLAGS	+= $(CC_FLAGS_SCS)
+export CC_FLAGS_SCS
+endif
+
 # arch Makefile may override CC so keep this after arch Makefile is included
 NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
 
arch/Kconfig +34 -0
index 98de654b79b3..a67fa78c92e7 100644
@@ -526,6 +526,40 @@ config STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config ARCH_SUPPORTS_SHADOW_CALL_STACK
+	bool
+	help
+	  An architecture should select this if it supports Clang's Shadow
+	  Call Stack, has asm/scs.h, and implements runtime support for shadow
+	  stack switching.
+
+config SHADOW_CALL_STACK
+	bool "Clang Shadow Call Stack"
+	depends on ARCH_SUPPORTS_SHADOW_CALL_STACK
+	help
+	  This option enables Clang's Shadow Call Stack, which uses a
+	  shadow stack to protect function return addresses from being
+	  overwritten by an attacker. More information can be found in
+	  Clang's documentation:
+
+	    https://clang.llvm.org/docs/ShadowCallStack.html
+
+	  Note that security guarantees in the kernel differ from the ones
+	  documented for user space. The kernel must store addresses of shadow
+	  stacks used by other tasks and interrupt handlers in memory, which
+	  means an attacker capable of reading and writing arbitrary memory
+	  may be able to locate them and hijack control flow by modifying
+	  shadow stacks that are not currently in use.
+
+config SHADOW_CALL_STACK_VMAP
+	bool "Use virtually mapped shadow call stacks"
+	depends on SHADOW_CALL_STACK
+	help
+	  Use virtually mapped shadow call stacks. Selecting this option
+	  provides better stack exhaustion protection, but increases per-thread
+	  memory consumption as a full page is allocated for each shadow stack.
+
+
 config HAVE_ARCH_WITHIN_STACK_FRAMES
 	bool
 	help
include/linux/compiler-clang.h +6 -0
index 333a6695a918..18fc4d29ef27 100644
@@ -42,3 +42,9 @@
  * compilers, like ICC.
  */
 #define barrier() __asm__ __volatile__("" : : : "memory")
+
+#if __has_feature(shadow_call_stack)
+# define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
+#else
+# define __noscs
+#endif
include/linux/compiler_types.h +4 -0
index 72393a8c1a6c..be5d5be4b1ae 100644
@@ -202,6 +202,10 @@ struct ftrace_likely_data {
 # define randomized_struct_fields_end
 #endif
 
+#ifndef __noscs
+# define __noscs
+#endif
+
 #ifndef asm_volatile_goto
 #define asm_volatile_goto(x...) asm goto(x)
 #endif
include/linux/scs.h new file +57 -0

57 lines not on this page · raw message

init/init_task.c +8 -0
index 9e5cbe5eab7b..cbd40460e903 100644
@@ -11,6 +11,7 @@
 #include <linux/mm.h>
 #include <linux/audit.h>
 #include <linux/numa.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <linux/uaccess.h>
@@ -184,6 +185,13 @@ struct task_struct init_task
 };
 EXPORT_SYMBOL(init_task);
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] __init_task_data
+		__aligned(SCS_SIZE) = {
+	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
+};
+#endif
+
 /*
  * Initial thread structure. Alignment of this is handled by a special
  * linker map entry.
kernel/Makefile +1 -0
index 4cb4130ced32..c332eb9d4841 100644
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/fork.c +9 -0
index 60a1295f4384..2bc73d654593 100644
@@ -94,6 +94,7 @@
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
 #include <linux/kasan.h>
+#include <linux/scs.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -454,6 +455,8 @@ void put_task_stack(struct task_struct *tsk)
 
 void free_task(struct task_struct *tsk)
 {
+	scs_release(tsk);
+
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	/*
 	 * The task is finally done with both the stack and thread_info,
@@ -837,6 +840,8 @@ void __init fork_init(void)
 			  NULL, free_vm_stack_cache);
 #endif
 
+	scs_init();
+
 	lockdep_init_task(&init_task);
 	uprobes_init();
 }
@@ -896,6 +901,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	if (err)
 		goto free_stack;
 
+	err = scs_prepare(tsk, node);
+	if (err)
+		goto free_stack;
+
 #ifdef CONFIG_SECCOMP
 	/*
 	 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c +2 -0
index 1a9983da4408..7473cd685560 100644
@@ -11,6 +11,7 @@
 #include <linux/nospec.h>
 
 #include <linux/kcov.h>
+#include <linux/scs.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
@@ -6036,6 +6037,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	idle->se.exec_start = sched_clock();
 	idle->flags |= PF_IDLE;
 
+	scs_task_reset(idle);
 	kasan_unpoison_task_stack(idle);
 
 #ifdef CONFIG_SMP
kernel/scs.c new file +187 -0

187 lines not on this page · raw message

Signature · 1 line
2.25.0.265.gbab2e86ba0-goog