Skip to contentSkip to conversation

stackleak plugin port to upstream kernel

kernel-hardening Tycho Andersen 2 participants 2 messages Mar 13, 2017 – Mar 14, 2017

Subsystems (6)
x86 VFS Kbuild exec and binfmt +2 more subsystems
lore
Share & tools
lore
#0 lore raw

Hi all,

I have an initial version of a port of the stackleak plugin ported to the mainline kernel (attached), but naturally it doesn't quite work, killing init with:

[    0.684209] Kernel BUG at ffffffff819893e2 [verbose debug info unavailable]
[    0.686467] invalid opcode: 0000 [#2] SMP
[    0.688337] Modules linked in:
[    0.691232] CPU: 3 PID: 1 Comm: init Tainted: G      D         4.11.0-rc1+ #5
[    0.693076] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu2 04/01/2014
[    0.695461] task: ffff880134af0000 task.stack: ffffc90000630000
[    0.696736] RIP: 0010:erase_kstack+0x52/0x80
[    0.697854] RSP: 0000:ffffc90000633f28 EFLAGS: 00010006
[    0.699025] RAX: ffffffffffff4111 RBX: ffffffff81982a20 RCX: 0000000000633f18
[    0.700430] RDX: 000000000000028b RSI: 0000000000000002 RDI: 0000000000000010
[    0.701989] RBP: ffffc90000633f50 R08: 000000000001cc00 R09: 0000000000000000
[    0.703693] R10: ffffea0004d1b780 R11: ffff880134af0000 R12: 0000000000000000
[    0.705069] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[    0.706393] FS:  0000000000000000(0000) GS:ffff880139d80000(0000) knlGS:0000000000000000
[    0.707958] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    0.708917] CR2: 00007ffe49fb2a29 CR3: 000000013476d000 CR4: 00000000000006e0
[    0.709946] Call Trace:
[    0.710565]  ? ret_from_fork+0x20/0x40
[    0.711285] Code: 03 f2 48 af 67 e3 12 83 f9 10 72 0d b9 10 00 00 00 f3 48 af 67 e3 02 75 dd fc 48 83 cf 10 89 e1 29 f9 48 81 f9 00 40 00 00 72 02 <0f> 0b c1 e9 03 f3 48 ab 49 8b bb d8 08 00 00 48 81 ef 00 01 00 
[    0.713935] RIP: erase_kstack+0x52/0x80 RSP: ffffc90000633f28

The problem seems to be in the erase_kstack routine in arch/x86/entry/entry_64.S, it seems to be looking for a series of 0xBEEFs, which aren't found. I'm struggling to figure out where these 0xBEEFs come from:
are they part of the mainline kernel stack initialization and something has gone totally haywire, or is this some PaX thing that I've overlooked?

Thanks!

Tycho

Quoted 1 line

From 27d444303e541c741d37ec9691c00f8b0e53d4d4 Mon Sep 17 00:00:00 2001

From: Tycho Andersen <[email protected]>
Date: Thu, 9 Mar 2017 17:47:46 -0800
Subject: [PATCH] gcc-plugins: add stackleak plugin to zero kernel stack

This plugin adds an option to zero the kernel stack after a syscall return so that no information is leaked on the kernel stack.

Ported from grsecurity/PaX. Minor changes to macro names to be more upstream-esque.

---
 arch/Kconfig                           |  15 ++
 arch/x86/Kconfig                       |   1 +
 arch/x86/entry/common.c                |  15 +-
 arch/x86/entry/entry_32.S              |  68 +++++++
 arch/x86/entry/entry_64.S              |  69 +++++++
 arch/x86/entry/entry_64_compat.S       |  12 ++
 arch/x86/include/asm/processor.h       |   4 +
 arch/x86/kernel/asm-offsets.c          |   9 +
 arch/x86/kernel/dumpstack_64.c         |  33 ++++
 fs/exec.c                              |  13 ++
 scripts/Makefile.gcc-plugins           |   3 +
 scripts/gcc-plugins/stackleak_plugin.c | 342 +++++++++++++++++++++++++++++++++
 12 files changed, 582 insertions(+), 2 deletions(-)
 create mode 100644 scripts/gcc-plugins/stackleak_plugin.c
arch/Kconfig +15 -0
index cd211a1..48af925 100644
@@ -436,6 +436,21 @@ config GCC_PLUGIN_STRUCTLEAK_VERBOSE
 	  initialized. Since not all existing initializers are detected
 	  by the plugin, this can produce false positive warnings.
 
+config ARCH_HAS_STACKLEAK
+	def_bool n
+
+config GCC_PLUGIN_STACKLEAK
+	bool "Zero the kernel stack after syscalls"
+	depends on GCC_PLUGINS
+	depends on ARCH_HAS_STACKLEAK
+	help
+	  This plugin zero-initializes the kernel stack after every system call. This
+	  can prevent some classes of information exposures.
+
+	  This plugin was ported from grsecurity/PaX. More information at:
+	   * https://grsecurity.net/
+	   * https://pax.grsecurity.net/
+
 config HAVE_CC_STACKPROTECTOR
 	bool
 	help
arch/x86/Kconfig +1 -0
index cc98d5a..5988a5f 100644
@@ -56,6 +56,7 @@ config X86
 	select ARCH_HAS_PMEM_API		if X86_64
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_SG_CHAIN
+	select ARCH_HAS_STACKLEAK
 	select ARCH_HAS_STRICT_KERNEL_RWX
 	select ARCH_HAS_STRICT_MODULE_RWX
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
arch/x86/entry/common.c +13 -2
index 370c42c..39cb5fd 100644
@@ -43,6 +43,12 @@ __visible inline void enter_from_user_mode(void)
 static inline void enter_from_user_mode(void) {}
 #endif
 
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+asmlinkage void erase_kstack(void);
+#else
+static void erase_kstack(void) {}
+#endif
+
 static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
 {
 #ifdef CONFIG_X86_64
@@ -79,8 +85,10 @@ static long syscall_trace_enter(struct pt_regs *regs)
 		emulated = true;
 
 	if ((emulated || (work & _TIF_SYSCALL_TRACE)) &&
-	    tracehook_report_syscall_entry(regs))
+	    tracehook_report_syscall_entry(regs)) {
+		erase_kstack();
 		return -1L;
+	}
 
 	if (emulated)
 		return -1L;
@@ -115,8 +123,10 @@ static long syscall_trace_enter(struct pt_regs *regs)
 		}
 
 		ret = __secure_computing(&sd);
-		if (ret == -1)
+		if (ret == -1) {
+			erase_kstack();
 			return ret;
+		}
 	}
 #endif
 
@@ -125,6 +135,7 @@ static long syscall_trace_enter(struct pt_regs *regs)
 
 	do_audit_syscall_entry(regs, arch);
 
+	erase_kstack();
 	return ret ?: regs->orig_ax;
 }
 
arch/x86/entry/entry_32.S +68 -0

86 lines not on this page · raw message

arch/x86/entry/entry_64.S +69 -0

99 lines not on this page · raw message

arch/x86/entry/entry_64_compat.S +12 -0
index e1721da..63af1d5 100644
@@ -18,6 +18,12 @@
 
 	.section .entry.text, "ax"
 
+	.macro erase_kstack
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	call erase_kstack
+#endif
+	.endm
+
 /*
  * 32-bit SYSENTER entry.
  *
@@ -229,6 +235,9 @@ ENTRY(entry_SYSCALL_compat)
 
 	/* Opportunistic SYSRET */
 sysret32_from_system_call:
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	erase_kstack
+#endif
 	TRACE_IRQS_ON			/* User mode traces as IRQs on. */
 	movq	RBX(%rsp), %rbx		/* pt_regs->rbx */
 	movq	RBP(%rsp), %rbp		/* pt_regs->rbp */
@@ -337,6 +346,9 @@ ENTRY(entry_INT80_compat)
 .Lsyscall_32_done:
 
 	/* Go back to user mode. */
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	erase_kstack
+#endif
 	TRACE_IRQS_ON
 	SWAPGS
 	jmp	restore_regs_and_iret
arch/x86/include/asm/processor.h +4 -0
index f385eca..e47c9bd 100644
@@ -469,6 +469,10 @@ struct thread_struct {
 
 	mm_segment_t		addr_limit;
 
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	unsigned long		lowest_stack;
+#endif
+
 	unsigned int		sig_on_uaccess_err:1;
 	unsigned int		uaccess_err:1;	/* uaccess failed */
 
arch/x86/kernel/asm-offsets.c +9 -0
index de827d6..4ed7451 100644
@@ -37,6 +37,10 @@ void common(void) {
 	BLANK();
 	OFFSET(TASK_TI_flags, task_struct, thread_info.flags);
 	OFFSET(TASK_addr_limit, task_struct, thread.addr_limit);
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	OFFSET(TASK_lowest_stack, task_struct, thread.lowest_stack);
+	OFFSET(TASK_thread_sp0, task_struct, thread.sp0);
+#endif
 
 	BLANK();
 	OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
@@ -73,6 +77,11 @@ void common(void) {
 	OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2);
 #endif
 
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+	BLANK();
+	DEFINE(THREAD_SIZE_asm, THREAD_SIZE);
+#endif
+
 #ifdef CONFIG_XEN
 	BLANK();
 	OFFSET(XEN_vcpu_info_mask, vcpu_info, evtchn_upcall_mask);
arch/x86/kernel/dumpstack_64.c +33 -0
index a8b117e..dc29be8 100644
@@ -188,3 +188,36 @@ int is_valid_bugaddr(unsigned long ip)
 
 	return ud2 == 0x0b0f;
 }
+
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+void __used check_alloca(unsigned long size)
+{
+	struct stack_info stack_info = {0};
+	unsigned long visit_mask = 0;
+	unsigned long sp = (unsigned long)&sp;
+	unsigned long stack_left;
+
+	BUG_ON(get_stack_info(&sp, current, &stack_info, &visit_mask));
+
+	switch (stack_info.type) {
+	case STACK_TYPE_TASK:
+		stack_left = sp & (THREAD_SIZE - 1);
+		break;
+
+	case STACK_TYPE_IRQ:
+		stack_left = sp & (IRQ_STACK_SIZE - 1);
+		break;
+
+	case STACK_TYPE_EXCEPTION ... STACK_TYPE_EXCEPTION_LAST:
+		stack_left = sp & (EXCEPTION_STKSZ - 1);
+		break;
+
+	case STACK_TYPE_SOFTIRQ:
+	default:
+		BUG();
+	}
+
+	BUG_ON(stack_left < 256 || size >= stack_left - 256);
+}
+EXPORT_SYMBOL(check_alloca);
+#endif
fs/exec.c +13 -0
index 65145a3..a116bd20 100644
@@ -1927,3 +1927,16 @@ COMPAT_SYSCALL_DEFINE5(execveat, int, fd,
 				  argv, envp, flags);
 }
 #endif
+
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
+void __used track_stack(void)
+{
+	unsigned long sp = (unsigned long)&sp;
+	if (sp < current->thread.lowest_stack &&
+	    sp >= (unsigned long)task_stack_page(current) + 2 * sizeof(unsigned long))
+		current->thread.lowest_stack = sp;
+	if (unlikely((sp & ~(THREAD_SIZE - 1)) < (THREAD_SIZE/16)))
+		BUG();
+}
+EXPORT_SYMBOL(track_stack);
+#endif
scripts/Makefile.gcc-plugins +3 -0
index 8233553..c998092 100644
@@ -29,6 +29,9 @@ ifdef CONFIG_GCC_PLUGINS
   gcc-plugin-cflags-$(CONFIG_GCC_PLUGIN_STRUCTLEAK_VERBOSE)	+= -fplugin-arg-structleak_plugin-verbose
   gcc-plugin-cflags-$(CONFIG_GCC_PLUGIN_STRUCTLEAK)	+= -DSTRUCTLEAK_PLUGIN
 
+  gcc-plugin-$(CONFIG_PAX_MEMORY_STACKLEAK)		+= stackleak_plugin.so$
+  gcc-plugin-cflags-$(CONFIG_PAX_MEMORY_STACKLEAK)	+= -DSTACKLEAK_PLUGIN -fplugin-arg-stackleak_plugin-track-lowest-sp=100$
+
   GCC_PLUGINS_CFLAGS := $(strip $(addprefix -fplugin=$(objtree)/scripts/gcc-plugins/, $(gcc-plugin-y)) $(gcc-plugin-cflags-y))
 
   export PLUGINCC GCC_PLUGINS_CFLAGS GCC_PLUGIN GCC_PLUGIN_SUBDIR
scripts/gcc-plugins/stackleak_plugin.c new file +342 -0

342 lines not on this page · raw message

Signature · 1 line
2.9.3
#1 lore raw

Yay more plugins! :)

In the future please include patches in-line... it's hard to comment on them if they're attachments. More comments below...

Quoted 5 lines · Tycho Andersen

The problem seems to be in the erase_kstack routine in arch/x86/entry/entry_64.S, it seems to be looking for a series of 0xBEEFs, which aren't found. I'm struggling to figure out where these 0xBEEFs come from:
are they part of the mainline kernel stack initialization and something has gone totally haywire, or is this some PaX thing that I've overlooked?

It looks like the Kconfig renaming wasn't complete, and your build isn't building/running the plugin at all:

arch/Kconfig config GCC_PLUGIN_STRUCTLEAK_VERBOSE

Quoted 5 lines
arch/Kconfig +4 -0
+config GCC_PLUGIN_STACKLEAK
+       bool "Zero the kernel stack after syscalls"
[...]
+  gcc-plugin-$(CONFIG_PAX_MEMORY_STACKLEAK)            += stackleak_plugin.so$
+  gcc-plugin-cflags-$(CONFIG_PAX_MEMORY_STACKLEAK)     += -DSTACKLEAK_PLUGIN -fplugin-arg-stackleak_plugin-track-lowest-sp=100$

(CONFIG_GCC_PLUGIN_STACKLEAK vs CONFIG_PAX_MEMORY_STACKLEAK)

Also, becareful of #ifdef CONFIG_GCC_PLUGIN_STACKLEAK vs #ifdef STACKLEAK_PLUGIN. That has gotten confused with some other plugin ports too.

Otherwise, this looks good for an initial extraction. For the next version there should probably be more comments on the new functions for what they do, and detailing for the ARCH_HAS Kconfig for what an architecture needs to do to correctly implement it (see examples for seccomp, hardened usercopy, etc), and an attempt made to reduce the number of #ifdef blocks. For example, instead of:

#ifdef THE_THING
     do_the_thing();

#endif

Build the definition of do_the_thing() with the #ifdef in one place. In C, this would be:

#ifdef THE_THING
void do_the_thing();
#else
static inline void do_the_thing() { }
#endif

Then do_the_thing() can be included everywhere without the #ifdef block. (I haven't spent much time looking at what's needed in .S files to make this happen, but I'm sure there is a similar pattern.)

Oh, and we should probably build a test for this, though lkdtm doesn't seem like the right place since it's expecting a Bug-style protection. Maybe something more like lib/test_user_copy.c that just exercises stuff to see if it's correctly behaving?

Thanks for working on this and sending this for people to look at!

-Kees

Signature · 2 lines
Kees Cook
Pixel Security