Skip to contentSkip to conversation

x86-64: Stack protector and percpu improvements

PATCH Superseded Brian Gerst 2 participants 14 messages Oct 23, 2023 – Oct 24, 2023

Subsystems (6)

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

cover x86-64: Stack protector and percpu improvements

Currently, x86-64 uses an unusual per-cpu layout, where the percpu section is linked at absolute address 0. The reason behind this is that older GCC versions placed the stack protector (if enabled) at a fixed offset from the GS segment base. Since the GS segement is also used for percpu variables, this forced the current layout.

GCC since version 8.1 supports a configurable location for the stack protector value, which allows removal of the restriction on how the percpu section is linked. This allows the percpu section to be linked normally, like most other architectures. In turn, this allows removal of code that was needed to support the zero-based percpu section.

P.S.: I posted this series back in 2021, and at the time it appeared that GCC 8.1 was too recent (released May 2, 2018) to be a requirement to keep stack protector support. Two years later, support for the new compiler options should be much more widespread.

Brian Gerst (9):
x86/stackprotector/32: Remove stack protector test script x86/boot: Disable stack protector for early boot code x86/stackprotector/64: Convert stack protector to normal percpu
variable
x86/percpu/64: Remove fixed_percpu_data x86/percpu/64: Use relative percpu offsets x86/boot/64: Remove inverse relocations x86/percpu/64: Remove INIT_PER_CPU macros
percpu: Remove PER_CPU_FIRST_SECTION
kallsyms: Remove KALLSYMS_ABSOLUTE_PERCPU

 arch/x86/Kconfig                          |   7 +-
 arch/x86/Makefile                         |  19 +--
 arch/x86/boot/compressed/misc.c           |  14 +--
 arch/x86/entry/entry_64.S                 |   2 +-
 arch/x86/include/asm/percpu.h             |  22 ----
 arch/x86/include/asm/processor.h          |  28 +----
 arch/x86/include/asm/stackprotector.h     |  37 ++----
 arch/x86/kernel/Makefile                  |   2 +
 arch/x86/kernel/asm-offsets_64.c          |   6 -
 arch/x86/kernel/cpu/common.c              |   8 +-
 arch/x86/kernel/head_64.S                 |  10 +-
 arch/x86/kernel/irq_64.c                  |   1 -
 arch/x86/kernel/setup_percpu.c            |  12 +-
 arch/x86/kernel/vmlinux.lds.S             |  35 ------
 arch/x86/tools/relocs.c                   | 136 +---------------------
 arch/x86/xen/xen-head.S                   |   6 +-
 include/asm-generic/vmlinux.lds.h         |   1 -
 include/linux/percpu-defs.h               |  12 --
 init/Kconfig                              |  11 +-
 kernel/kallsyms.c                         |  12 +-
 scripts/gcc-x86_32-has-stack-protector.sh |   8 --
 scripts/gcc-x86_64-has-stack-protector.sh |   4 -
 scripts/kallsyms.c                        |  80 +++----------
 scripts/link-vmlinux.sh                   |   4 -
 24 files changed, 60 insertions(+), 417 deletions(-)
 delete mode 100755 scripts/gcc-x86_32-has-stack-protector.sh
 delete mode 100755 scripts/gcc-x86_64-has-stack-protector.sh
Signature · 1 line
2.41.0
#1 lore raw

PATCH 1/9 x86/stackprotector/32: Remove stack protector test script

Test for compiler support directly in Kconfig.

---
 arch/x86/Kconfig                          | 2 +-
 scripts/gcc-x86_32-has-stack-protector.sh | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)
 delete mode 100755 scripts/gcc-x86_32-has-stack-protector.sh
arch/x86/Kconfig +1 -1
index 5ef081aa12ac..039872be1630 100644
@@ -411,7 +411,7 @@ config PGTABLE_LEVELS
 config CC_HAS_SANE_STACKPROTECTOR
 	bool
 	default $(success,$(srctree)/scripts/gcc-x86_64-has-stack-protector.sh $(CC) $(CLANG_FLAGS)) if 64BIT
-	default $(success,$(srctree)/scripts/gcc-x86_32-has-stack-protector.sh $(CC) $(CLANG_FLAGS))
+	default $(cc-option,-mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard)
 	help
 	  We have to make sure stack protector is unconditionally disabled if
 	  the compiler produces broken code or if it does not let us control
scripts/gcc-x86_32-has-stack-protector.sh deleted +0 -8
deleted file mode 100755
index 825c75c5b715..000000000000
@@ -1,8 +0,0 @@
-#!/bin/sh
-# SPDX-License-Identifier: GPL-2.0
-
-# This requires GCC 8.1 or better.  Specifically, we require
-# -mstack-protector-guard-reg, added by
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81708
-
-echo "int foo(void) { char X[200]; return 3; }" | $* -S -x c -c -m32 -O0 -fstack-protector -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard - -o - 2> /dev/null | grep -q "%fs"
Signature · 1 line
2.41.0
#2 lore raw

PATCH 2/9 x86/boot: Disable stack protector for early boot code

In reply to Brian Gerst

On 64-bit, this will prevent crashes when the canary access is changed from %gs:40 to %gs:__stack_chk_guard(%rip). RIP-relative addresses from the identity-mapped early boot code will target the wrong address with zero-based percpu. KASLR could then shift that address to an unmapped page causing a crash on boot.

This early boot code runs well before userspace is active and does not need stack protector enabled.

---
 arch/x86/kernel/Makefile | 2 ++
 1 file changed, 2 insertions(+)
arch/x86/kernel/Makefile +2 -0
index 0000325ab98f..aff619054e17 100644
@@ -39,6 +39,8 @@ KMSAN_SANITIZE_nmi.o					:= n
 KCOV_INSTRUMENT_head$(BITS).o				:= n
 KCOV_INSTRUMENT_sev.o					:= n
 
+CFLAGS_head32.o := -fno-stack-protector
+CFLAGS_head64.o := -fno-stack-protector
 CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace
 
 obj-y			+= head_$(BITS).o
Signature · 1 line
2.41.0
#5 lore raw

PATCH 4/9 x86/percpu/64: Remove fixed_percpu_data

In reply to Brian Gerst

Now that the stack protector canary value is a normal percpu variable, fixed_percpu_data is unused and can be removed.

---
 arch/x86/include/asm/processor.h | 13 +++++--------
 arch/x86/kernel/cpu/common.c     |  4 ----
 arch/x86/kernel/head_64.S        | 12 ++++++------
 arch/x86/kernel/vmlinux.lds.S    |  6 ------
 arch/x86/tools/relocs.c          |  1 -
 arch/x86/xen/xen-head.S          | 12 ++++++++----
 6 files changed, 19 insertions(+), 29 deletions(-)
arch/x86/include/asm/processor.h +5 -8
index 04371f60e3c6..48c31b8e3e72 100644
@@ -393,16 +393,13 @@ struct irq_stack {
 } __aligned(IRQ_STACK_SIZE);
 
 #ifdef CONFIG_X86_64
-struct fixed_percpu_data {
-	char		gs_base[40];
-};
-
-DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
-DECLARE_INIT_PER_CPU(fixed_percpu_data);
-
 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
 {
-	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
+#ifdef CONFIG_SMP
+	return per_cpu_offset(cpu);
+#else
+	return 0;
+#endif
 }
 
 extern asmlinkage void entry_SYSCALL32_ignore(void);
arch/x86/kernel/cpu/common.c +0 -4
index f9c8bd27b642..a44fd3ad460e 100644
@@ -2051,10 +2051,6 @@ DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
 EXPORT_PER_CPU_SYMBOL(pcpu_hot);
 
 #ifdef CONFIG_X86_64
-DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
-		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
-EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
-
 static void wrmsrl_cstar(unsigned long val)
 {
 	/*
arch/x86/kernel/head_64.S +6 -6
index 3dcabbc49149..f2453eb38417 100644
@@ -72,9 +72,14 @@ SYM_CODE_START_NOALIGN(startup_64)
 
 	/* Setup GSBASE to allow stack canary access for C code */
 	movl	$MSR_GS_BASE, %ecx
-	leaq	INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
+#ifdef CONFIG_SMP
+	leaq	__per_cpu_load(%rip), %rdx
 	movl	%edx, %eax
 	shrq	$32,  %rdx
+#else
+	xorl	%eax, %eax
+	xorl	%edx, %edx
+#endif
 	wrmsr
 
 	call	startup_64_setup_env
@@ -345,15 +350,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 
 	/* Set up %gs.
 	 *
-	 * The base of %gs always points to fixed_percpu_data. If the
-	 * stack protector canary is enabled, it is located at %gs:40.
 	 * Note that, on SMP, the boot cpu uses init data section until
 	 * the per cpu areas are set up.
 	 */
 	movl	$MSR_GS_BASE,%ecx
-#ifndef CONFIG_SMP
-	leaq	INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
-#endif
 	movl	%edx, %eax
 	shrq	$32, %rdx
 	wrmsr
arch/x86/kernel/vmlinux.lds.S +0 -6
index 54a5596adaa6..c87dc8de2084 100644
@@ -509,14 +509,8 @@ SECTIONS
  */
 #define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
 INIT_PER_CPU(gdt_page);
-INIT_PER_CPU(fixed_percpu_data);
 INIT_PER_CPU(irq_stack_backing_store);
 
-#ifdef CONFIG_SMP
-. = ASSERT((fixed_percpu_data == 0),
-           "fixed_percpu_data is not at start of per-cpu area");
-#endif
-
 #ifdef CONFIG_CPU_UNRET_ENTRY
 . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
 #endif
arch/x86/tools/relocs.c +0 -1
index d30949e25ebd..3ccd9d4fcf9c 100644
@@ -811,7 +811,6 @@ static void percpu_init(void)
  *	__per_cpu_load
  *
  * The "gold" linker incorrectly associates:
- *	init_per_cpu__fixed_percpu_data
  *	init_per_cpu__gdt_page
  */
 static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
arch/x86/xen/xen-head.S +8 -4
index a0ea285878db..9ce0d9d268bb 100644
@@ -53,14 +53,18 @@ SYM_CODE_START(startup_xen)
 
 	/* Set up %gs.
 	 *
-	 * The base of %gs always points to fixed_percpu_data.  If the
-	 * stack protector canary is enabled, it is located at %gs:40.
 	 * Note that, on SMP, the boot cpu uses init data section until
 	 * the per cpu areas are set up.
 	 */
 	movl	$MSR_GS_BASE,%ecx
-	movq	$INIT_PER_CPU_VAR(fixed_percpu_data),%rax
-	cdq
+#ifdef CONFIG_SMP
+	leaq	__per_cpu_load(%rip), %rdx
+	movl	%edx, %eax
+	shrq	$32, %rdx
+#else
+	xorl	%eax, %eax
+	xorl	%edx, %edx
+#endif
 	wrmsr
 
 	mov	%rsi, %rdi
Signature · 1 line
2.41.0
#3 lore raw

PATCH 3/9 x86/stackprotector/64: Convert stack protector to normal percpu variable

In reply to Brian Gerst

Older versions of GCC fixed the location of the stack protector canary at %gs:40. This constraint forced the percpu section to be linked at virtual address 0 so that the canary could be the first data object in the percpu section. Supporting the zero-based percpu section requires additional code to handle relocations for RIP-relative references to percpu data, extra complexity to kallsyms, and workarounds for linker bugs due to the use of absolute symbols.

Since version 8.1, GCC has options to configure the location of the canary value. This allows the canary to be turned into a normal percpu variable and removes the constraint that the percpu section be zero-based.

---
 arch/x86/Kconfig                          |  5 ++-
 arch/x86/Makefile                         | 19 +++++++-----
 arch/x86/entry/entry_64.S                 |  2 +-
 arch/x86/include/asm/processor.h          | 15 +--------
 arch/x86/include/asm/stackprotector.h     | 37 +++++------------------
 arch/x86/kernel/asm-offsets_64.c          |  6 ----
 arch/x86/kernel/cpu/common.c              |  4 +--
 scripts/gcc-x86_64-has-stack-protector.sh |  4 ---
 8 files changed, 24 insertions(+), 68 deletions(-)
 delete mode 100755 scripts/gcc-x86_64-has-stack-protector.sh
arch/x86/Kconfig +2 -3
index 039872be1630..c95e0ce557da 100644
@@ -410,12 +410,11 @@ config PGTABLE_LEVELS
 
 config CC_HAS_SANE_STACKPROTECTOR
 	bool
-	default $(success,$(srctree)/scripts/gcc-x86_64-has-stack-protector.sh $(CC) $(CLANG_FLAGS)) if 64BIT
+	default $(cc-option,-mstack-protector-guard-reg=gs -mstack-protector-guard-symbol=__stack_chk_guard) if 64BIT
 	default $(cc-option,-mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard)
 	help
 	  We have to make sure stack protector is unconditionally disabled if
-	  the compiler produces broken code or if it does not let us control
-	  the segment on 32-bit kernels.
+	  the compiler does not allow control of the segment and symbol.
 
 menu "Processor type and features"
 
arch/x86/Makefile +12 -7
index 22e41d9dbc23..169f086997ec 100644
@@ -111,13 +111,7 @@ ifeq ($(CONFIG_X86_32),y)
         # temporary until string.h is fixed
         KBUILD_CFLAGS += -ffreestanding
 
-	ifeq ($(CONFIG_STACKPROTECTOR),y)
-		ifeq ($(CONFIG_SMP),y)
-			KBUILD_CFLAGS += -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard
-		else
-			KBUILD_CFLAGS += -mstack-protector-guard=global
-		endif
-	endif
+	percpu_seg := fs
 else
         BITS := 64
         UTS_MACHINE := x86_64
@@ -167,6 +161,17 @@ else
         KBUILD_CFLAGS += -mcmodel=kernel
         KBUILD_RUSTFLAGS += -Cno-redzone=y
         KBUILD_RUSTFLAGS += -Ccode-model=kernel
+
+	percpu_seg := gs
+endif
+
+ifeq ($(CONFIG_STACKPROTECTOR),y)
+	ifeq ($(CONFIG_SMP),y)
+		KBUILD_CFLAGS += -mstack-protector-guard-reg=$(percpu_seg) \
+				 -mstack-protector-guard-symbol=__stack_chk_guard
+	else
+		KBUILD_CFLAGS += -mstack-protector-guard=global
+	endif
 endif
 
 #
arch/x86/entry/entry_64.S +1 -1
index 1a88ad8a7b48..cddcc236aaae 100644
@@ -190,7 +190,7 @@ SYM_FUNC_START(__switch_to_asm)
 
 #ifdef CONFIG_STACKPROTECTOR
 	movq	TASK_stack_canary(%rsi), %rbx
-	movq	%rbx, PER_CPU_VAR(fixed_percpu_data + FIXED_stack_canary)
+	movq	%rbx, PER_CPU_VAR(__stack_chk_guard)
 #endif
 
 	/*
arch/x86/include/asm/processor.h +1 -14
index ae81a7191c1c..04371f60e3c6 100644
@@ -394,16 +394,7 @@ struct irq_stack {
 
 #ifdef CONFIG_X86_64
 struct fixed_percpu_data {
-	/*
-	 * GCC hardcodes the stack canary as %gs:40.  Since the
-	 * irq_stack is the object at %gs:0, we reserve the bottom
-	 * 48 bytes of the irq stack for the canary.
-	 *
-	 * Once we are willing to require -mstack-protector-guard-symbol=
-	 * support for x86_64 stackprotector, we can get rid of this.
-	 */
 	char		gs_base[40];
-	unsigned long	stack_canary;
 };
 
 DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
@@ -418,11 +409,7 @@ extern asmlinkage void entry_SYSCALL32_ignore(void);
 
 /* Save actual FS/GS selectors and bases to current->thread */
 void current_save_fsgs(void);
-#else	/* X86_64 */
-#ifdef CONFIG_STACKPROTECTOR
-DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
-#endif
-#endif	/* !X86_64 */
+#endif /* X86_64 */
 
 struct perf_event;
 
arch/x86/include/asm/stackprotector.h +7 -30
index 00473a650f51..33abbd29ea26 100644
@@ -2,26 +2,13 @@
 /*
  * GCC stack protector support.
  *
- * Stack protector works by putting predefined pattern at the start of
+ * Stack protector works by putting a predefined pattern at the start of
  * the stack frame and verifying that it hasn't been overwritten when
- * returning from the function.  The pattern is called stack canary
- * and unfortunately gcc historically required it to be at a fixed offset
- * from the percpu segment base.  On x86_64, the offset is 40 bytes.
+ * returning from the function.  The pattern is called the stack canary
+ * and is a unique value for each task.
  *
- * The same segment is shared by percpu area and stack canary.  On
- * x86_64, percpu symbols are zero based and %gs (64-bit) points to the
- * base of percpu area.  The first occupant of the percpu area is always
- * fixed_percpu_data which contains stack_canary at the appropriate
- * offset.  On x86_32, the stack canary is just a regular percpu
- * variable.
- *
- * Putting percpu data in %fs on 32-bit is a minor optimization compared to
- * using %gs.  Since 32-bit userspace normally has %fs == 0, we are likely
- * to load 0 into %fs on exit to usermode, whereas with percpu data in
- * %gs, we are likely to load a non-null %gs on return to user mode.
- *
- * Once we are willing to require GCC 8.1 or better for 64-bit stackprotector
- * support, we can remove some of this complexity.
+ * GCC is configured to read the stack canary value from the __stack_chk_guard
+ * per-cpu variable, which is changed on task switch.
  */
 
 #ifndef _ASM_STACKPROTECTOR_H
@@ -36,6 +23,8 @@
 
 #include <linux/sched.h>
 
+DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
+
 /*
  * Initialize the stackprotector canary value.
  *
@@ -51,25 +40,13 @@ static __always_inline void boot_init_stack_canary(void)
 {
 	unsigned long canary = get_random_canary();
 
-#ifdef CONFIG_X86_64
-	BUILD_BUG_ON(offsetof(struct fixed_percpu_data, stack_canary) != 40);
-#endif
-
 	current->stack_canary = canary;
-#ifdef CONFIG_X86_64
-	this_cpu_write(fixed_percpu_data.stack_canary, canary);
-#else
 	this_cpu_write(__stack_chk_guard, canary);
-#endif
 }
 
 static inline void cpu_init_stack_canary(int cpu, struct task_struct *idle)
 {
-#ifdef CONFIG_X86_64
-	per_cpu(fixed_percpu_data.stack_canary, cpu) = idle->stack_canary;
-#else
 	per_cpu(__stack_chk_guard, cpu) = idle->stack_canary;
-#endif
 }
 
 #else	/* STACKPROTECTOR */
arch/x86/kernel/asm-offsets_64.c +0 -6
index bb65371ea9df..590b6cd0eac0 100644
@@ -54,11 +54,5 @@ int main(void)
 	BLANK();
 #undef ENTRY
 
-	BLANK();
-
-#ifdef CONFIG_STACKPROTECTOR
-	OFFSET(FIXED_stack_canary, fixed_percpu_data, stack_canary);
-	BLANK();
-#endif
 	return 0;
 }
arch/x86/kernel/cpu/common.c +1 -3
index b14fc8c1c953..f9c8bd27b642 100644
@@ -2103,15 +2103,13 @@ void syscall_init(void)
 	       X86_EFLAGS_AC|X86_EFLAGS_ID);
 }
 
-#else	/* CONFIG_X86_64 */
+#endif	/* CONFIG_X86_64 */
 
 #ifdef CONFIG_STACKPROTECTOR
 DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
 #endif
 
-#endif	/* CONFIG_X86_64 */
-
 /*
  * Clear all 6 debug registers:
  */
scripts/gcc-x86_64-has-stack-protector.sh deleted +0 -4
deleted file mode 100755
index 75e4e22b986a..000000000000
@@ -1,4 +0,0 @@
-#!/bin/sh
-# SPDX-License-Identifier: GPL-2.0
-
-echo "int foo(void) { char X[200]; return 3; }" | $* -S -x c -c -m64 -O0 -mcmodel=kernel -fno-PIE -fstack-protector - -o - 2> /dev/null | grep -q "%gs"
Signature · 1 line
2.41.0
#8 lore raw

PATCH 5/9 x86/percpu/64: Use relative percpu offsets

In reply to Brian Gerst

The percpu section is currently linked at virtual address 0, because older compilers hardcoded the stack protector canary value at a fixed offset from the start of the GS segment. Now that the canary is a normal percpu variable, the percpu section can be linked normally. This means that x86-64 will calculate percpu offsets like most other architectures, as the delta between the initial percpu address and the dynamically allocated memory.

---
 arch/x86/kernel/head_64.S      |  6 ------
 arch/x86/kernel/setup_percpu.c | 12 ++----------
 arch/x86/kernel/vmlinux.lds.S  | 24 +-----------------------
 arch/x86/tools/relocs.c        | 10 +++-------
 arch/x86/xen/xen-head.S        |  6 ------
 init/Kconfig                   |  2 +-
 6 files changed, 7 insertions(+), 53 deletions(-)
arch/x86/kernel/head_64.S +0 -6
index f2453eb38417..b35f74e58dd7 100644
@@ -72,14 +72,8 @@ SYM_CODE_START_NOALIGN(startup_64)
 
 	/* Setup GSBASE to allow stack canary access for C code */
 	movl	$MSR_GS_BASE, %ecx
-#ifdef CONFIG_SMP
-	leaq	__per_cpu_load(%rip), %rdx
-	movl	%edx, %eax
-	shrq	$32,  %rdx
-#else
 	xorl	%eax, %eax
 	xorl	%edx, %edx
-#endif
 	wrmsr
 
 	call	startup_64_setup_env
arch/x86/kernel/setup_percpu.c +2 -10
index 2c97bf7b56ae..8707dd07b9ce 100644
@@ -23,18 +23,10 @@
 #include <asm/cpumask.h>
 #include <asm/cpu.h>
 
-#ifdef CONFIG_X86_64
-#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
-#else
-#define BOOT_PERCPU_OFFSET 0
-#endif
-
-DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
 EXPORT_PER_CPU_SYMBOL(this_cpu_off);
 
-unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init = {
-	[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
-};
+unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init;
 EXPORT_SYMBOL(__per_cpu_offset);
 
 /*
arch/x86/kernel/vmlinux.lds.S +1 -23
index c87dc8de2084..8d8eb4d9ff9d 100644
@@ -102,12 +102,6 @@ jiffies = jiffies_64;
 PHDRS {
 	text PT_LOAD FLAGS(5);          /* R_E */
 	data PT_LOAD FLAGS(6);          /* RW_ */
-#ifdef CONFIG_X86_64
-#ifdef CONFIG_SMP
-	percpu PT_LOAD FLAGS(6);        /* RW_ */
-#endif
-	init PT_LOAD FLAGS(7);          /* RWE */
-#endif
 	note PT_NOTE FLAGS(0);          /* ___ */
 }
 
@@ -223,21 +217,7 @@ SECTIONS
 		__init_begin = .; /* paired with __init_end */
 	}
 
-#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
-	/*
-	 * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
-	 * output PHDR, so the next output section - .init.text - should
-	 * start another segment - init.
-	 */
-	PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu)
-	ASSERT(SIZEOF(.data..percpu) < CONFIG_PHYSICAL_START,
-	       "per-CPU data too large - increase CONFIG_PHYSICAL_START")
-#endif
-
 	INIT_TEXT_SECTION(PAGE_SIZE)
-#ifdef CONFIG_X86_64
-	:init
-#endif
 
 	/*
 	 * Section for code used exclusively before alternatives are run. All
@@ -367,9 +347,7 @@ SECTIONS
 		EXIT_DATA
 	}
 
-#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
 	PERCPU_SECTION(INTERNODE_CACHE_BYTES)
-#endif
 
 	. = ALIGN(PAGE_SIZE);
 
@@ -507,7 +485,7 @@ SECTIONS
  * Per-cpu symbols which need to be offset from __per_cpu_load
  * for the boot processor.
  */
-#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
+#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x)
 INIT_PER_CPU(gdt_page);
 INIT_PER_CPU(irq_stack_backing_store);
 
arch/x86/tools/relocs.c +3 -7
index 3ccd9d4fcf9c..01efbfdd3eb3 100644
@@ -815,12 +815,7 @@ static void percpu_init(void)
  */
 static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
 {
-	int shndx = sym_index(sym);
-
-	return (shndx == per_cpu_shndx) &&
-		strcmp(symname, "__init_begin") &&
-		strcmp(symname, "__per_cpu_load") &&
-		strncmp(symname, "init_per_cpu_", 13);
+	return 0;
 }
 
 
@@ -1043,7 +1038,8 @@ static int cmp_relocs(const void *va, const void *vb)
 
 static void sort_relocs(struct relocs *r)
 {
-	qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
+	if (r->count)
+		qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
 }
 
 static int write32(uint32_t v, FILE *f)
arch/x86/xen/xen-head.S +0 -6
index 9ce0d9d268bb..c1d9c92b417a 100644
@@ -57,14 +57,8 @@ SYM_CODE_START(startup_xen)
 	 * the per cpu areas are set up.
 	 */
 	movl	$MSR_GS_BASE,%ecx
-#ifdef CONFIG_SMP
-	leaq	__per_cpu_load(%rip), %rdx
-	movl	%edx, %eax
-	shrq	$32, %rdx
-#else
 	xorl	%eax, %eax
 	xorl	%edx, %edx
-#endif
 	wrmsr
 
 	mov	%rsi, %rdi
init/Kconfig +1 -1
index 6d35728b94b2..1af31b23e376 100644
@@ -1718,7 +1718,7 @@ config KALLSYMS_ALL
 config KALLSYMS_ABSOLUTE_PERCPU
 	bool
 	depends on KALLSYMS
-	default X86_64 && SMP
+	default n
 
 config KALLSYMS_BASE_RELATIVE
 	bool
Signature · 1 line
2.41.0
#10 lore raw

PATCH 6/9 x86/boot/64: Remove inverse relocations

In reply to Brian Gerst

Now that the percpu section is not at a fixed virtual address, inverse relocations, which were needed to offset the effects of relocation on RIP-relative percpu references, are no longer needed.

---
 arch/x86/boot/compressed/misc.c |  14 +---
 arch/x86/tools/relocs.c         | 126 +-------------------------------
 2 files changed, 2 insertions(+), 138 deletions(-)
arch/x86/boot/compressed/misc.c +1 -13
index b99e08e6815b..2de345a236c0 100644
@@ -221,7 +221,7 @@ static void handle_relocations(void *output, unsigned long output_len,
 
 	/*
 	 * Process relocations: 32 bit relocations first then 64 bit after.
-	 * Three sets of binary relocations are added to the end of the kernel
+	 * Two sets of binary relocations are added to the end of the kernel
 	 * before compression. Each relocation table entry is the kernel
 	 * address of the location which needs to be updated stored as a
 	 * 32-bit value which is sign extended to 64 bits.
@@ -231,8 +231,6 @@ static void handle_relocations(void *output, unsigned long output_len,
 	 * kernel bits...
 	 * 0 - zero terminator for 64 bit relocations
 	 * 64 bit relocation repeated
-	 * 0 - zero terminator for inverse 32 bit relocations
-	 * 32 bit inverse relocation repeated
 	 * 0 - zero terminator for 32 bit relocations
 	 * 32 bit relocation repeated
 	 *
@@ -249,16 +247,6 @@ static void handle_relocations(void *output, unsigned long output_len,
 		*(uint32_t *)ptr += delta;
 	}
 #ifdef CONFIG_X86_64
-	while (*--reloc) {
-		long extended = *reloc;
-		extended += map;
-
-		ptr = (unsigned long)extended;
-		if (ptr < min_addr || ptr > max_addr)
-			error("inverse 32-bit relocation outside of kernel!\n");
-
-		*(int32_t *)ptr -= delta;
-	}
 	for (reloc--; *reloc; reloc--) {
 		long extended = *reloc;
 		extended += map;
arch/x86/tools/relocs.c +1 -125

196 lines not on this page · raw message

Signature · 1 line
2.41.0
#11 lore raw

PATCH 7/9 x86/percpu/64: Remove INIT_PER_CPU macros

In reply to Brian Gerst

The load and link addresses of percpu variables are now the same, so these macros are no longer necessary.

---
 arch/x86/include/asm/percpu.h | 22 ----------------------
 arch/x86/kernel/irq_64.c      |  1 -
 arch/x86/kernel/vmlinux.lds.S |  7 -------
 arch/x86/tools/relocs.c       |  1 -
 4 files changed, 31 deletions(-)
arch/x86/include/asm/percpu.h +0 -22
index bbcc1ca737f0..11f50ab32d0e 100644
@@ -20,12 +20,6 @@
 
 #define PER_CPU_VAR(var)	__percpu(var)__percpu_rel
 
-#ifdef CONFIG_X86_64_SMP
-#define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
-#else
-#define INIT_PER_CPU_VAR(var)  var
-#endif
-
 #else /* ...!ASSEMBLY */
 
 #include <linux/kernel.h>
@@ -96,22 +90,6 @@
 #define __percpu_arg(x)		__percpu_prefix "%" #x
 #define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
 
-/*
- * Initialized pointers to per-cpu variables needed for the boot
- * processor need to use these macros to get the proper address
- * offset from __per_cpu_load on SMP.
- *
- * There also must be an entry in vmlinux_64.lds.S
- */
-#define DECLARE_INIT_PER_CPU(var) \
-       extern typeof(var) init_per_cpu_var(var)
-
-#ifdef CONFIG_X86_64_SMP
-#define init_per_cpu_var(var)  init_per_cpu__##var
-#else
-#define init_per_cpu_var(var)  var
-#endif
-
 /* For arch-specific code, we can use direct single-insn ops (they
  * don't give an lvalue though). */
 
arch/x86/kernel/irq_64.c +0 -1
index fe0c859873d1..30424f9876bc 100644
@@ -26,7 +26,6 @@
 #include <asm/apic.h>
 
 DEFINE_PER_CPU_PAGE_ALIGNED(struct irq_stack, irq_stack_backing_store) __visible;
-DECLARE_INIT_PER_CPU(irq_stack_backing_store);
 
 #ifdef CONFIG_VMAP_STACK
 /*
arch/x86/kernel/vmlinux.lds.S +0 -7
index 8d8eb4d9ff9d..089d164164e8 100644
@@ -481,13 +481,6 @@ SECTIONS
 	   "kernel image bigger than KERNEL_IMAGE_SIZE");
 
 #ifdef CONFIG_X86_64
-/*
- * Per-cpu symbols which need to be offset from __per_cpu_load
- * for the boot processor.
- */
-#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x)
-INIT_PER_CPU(gdt_page);
-INIT_PER_CPU(irq_stack_backing_store);
 
 #ifdef CONFIG_CPU_UNRET_ENTRY
 . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
arch/x86/tools/relocs.c +0 -1
index 7feb63179b62..931d90aa814c 100644
@@ -83,7 +83,6 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
 	"__initramfs_start|"
 	"(jiffies|jiffies_64)|"
 #if ELF_BITS == 64
-	"init_per_cpu__.*|"
 	"__end_rodata_hpage_align|"
 #endif
 	"__vvar_page|"
Signature · 1 line
2.41.0
#12 lore raw

PATCH 8/9 percpu: Remove PER_CPU_FIRST_SECTION

In reply to Brian Gerst

x86-64 was the only user.

---
 include/asm-generic/vmlinux.lds.h |  1 -
 include/linux/percpu-defs.h       | 12 ------------
 2 files changed, 13 deletions(-)
include/asm-generic/vmlinux.lds.h +0 -1
index 67d8dd2f1bde..23d8acc72760 100644
@@ -1032,7 +1032,6 @@
  */
 #define PERCPU_INPUT(cacheline)						\
 	__per_cpu_start = .;						\
-	*(.data..percpu..first)						\
 	. = ALIGN(PAGE_SIZE);						\
 	*(.data..percpu..page_aligned)					\
 	. = ALIGN(cacheline);						\
include/linux/percpu-defs.h +0 -12
index ec3573119923..b9ddee91e6c7 100644
@@ -26,13 +26,11 @@
 #define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
 #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
 #endif
-#define PER_CPU_FIRST_SECTION "..first"
 
 #else
 
 #define PER_CPU_SHARED_ALIGNED_SECTION ""
 #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
-#define PER_CPU_FIRST_SECTION ""
 
 #endif
 
@@ -114,16 +112,6 @@
 #define DEFINE_PER_CPU(type, name)					\
 	DEFINE_PER_CPU_SECTION(type, name, "")
 
-/*
- * Declaration/definition used for per-CPU variables that must come first in
- * the set of variables.
- */
-#define DECLARE_PER_CPU_FIRST(type, name)				\
-	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
-
-#define DEFINE_PER_CPU_FIRST(type, name)				\
-	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
-
 /*
  * Declaration/definition used for per-CPU variables that must be cacheline
  * aligned under SMP conditions so that, whilst a particular instance of the
Signature · 1 line
2.41.0
#13 lore raw

PATCH 9/9 kallsyms: Remove KALLSYMS_ABSOLUTE_PERCPU

In reply to Brian Gerst

x86-64 was the only user.

---
 init/Kconfig            | 11 +-----
 kernel/kallsyms.c       | 12 ++-----
 scripts/kallsyms.c      | 80 ++++++++---------------------------------
 scripts/link-vmlinux.sh |  4 ---
 4 files changed, 18 insertions(+), 89 deletions(-)
init/Kconfig +1 -10
index 1af31b23e376..4d91c5632aaf 100644
@@ -1715,11 +1715,6 @@ config KALLSYMS_ALL
 
 	  Say N unless you really need all symbols, or kernel live patching.
 
-config KALLSYMS_ABSOLUTE_PERCPU
-	bool
-	depends on KALLSYMS
-	default n
-
 config KALLSYMS_BASE_RELATIVE
 	bool
 	depends on KALLSYMS
@@ -1727,11 +1722,7 @@ config KALLSYMS_BASE_RELATIVE
 	help
 	  Instead of emitting them as absolute values in the native word size,
 	  emit the symbol references in the kallsyms table as 32-bit entries,
-	  each containing a relative value in the range [base, base + U32_MAX]
-	  or, when KALLSYMS_ABSOLUTE_PERCPU is in effect, each containing either
-	  an absolute value in the range [0, S32_MAX] or a relative value in the
-	  range [base, base + S32_MAX], where base is the lowest relative symbol
-	  address encountered in the image.
+	  each containing a relative value in the range [base, base + U32_MAX].
 
 	  On 64-bit builds, this reduces the size of the address table by 50%,
 	  but more importantly, it results in entries whose values are build
kernel/kallsyms.c +2 -10
index 18edd57b5fe8..f4e8e531052a 100644
@@ -151,16 +151,8 @@ unsigned long kallsyms_sym_address(int idx)
 	if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE))
 		return kallsyms_addresses[idx];
 
-	/* values are unsigned offsets if --absolute-percpu is not in effect */
-	if (!IS_ENABLED(CONFIG_KALLSYMS_ABSOLUTE_PERCPU))
-		return kallsyms_relative_base + (u32)kallsyms_offsets[idx];
-
-	/* ...otherwise, positive offsets are absolute values */
-	if (kallsyms_offsets[idx] >= 0)
-		return kallsyms_offsets[idx];
-
-	/* ...and negative offsets are relative to kallsyms_relative_base - 1 */
-	return kallsyms_relative_base - 1 - kallsyms_offsets[idx];
+	/* values are unsigned offsets */
+	return kallsyms_relative_base + (u32)kallsyms_offsets[idx];
 }
 
 static void cleanup_symbol_name(char *s)
scripts/kallsyms.c +15 -65

157 lines not on this page · raw message

scripts/link-vmlinux.sh +0 -4
index a432b171be82..d25b6d5de45e 100755
@@ -148,10 +148,6 @@ kallsyms()
 		kallsymopt="${kallsymopt} --all-symbols"
 	fi
 
-	if is_enabled CONFIG_KALLSYMS_ABSOLUTE_PERCPU; then
-		kallsymopt="${kallsymopt} --absolute-percpu"
-	fi
-
 	if is_enabled CONFIG_KALLSYMS_BASE_RELATIVE; then
 		kallsymopt="${kallsymopt} --base-relative"
 	fi
Signature · 1 line
2.41.0
#6 lore raw

Re: [PATCH 4/9] x86/percpu/64: Remove fixed_percpu_data

In reply to Brian Gerst
Quoted 139 lines · Brian Gerst

Now that the stack protector canary value is a normal percpu variable, fixed_percpu_data is unused and can be removed.

---
 arch/x86/include/asm/processor.h | 13 +++++--------
 arch/x86/kernel/cpu/common.c     |  4 ----
 arch/x86/kernel/head_64.S        | 12 ++++++------
 arch/x86/kernel/vmlinux.lds.S    |  6 ------
 arch/x86/tools/relocs.c          |  1 -
 arch/x86/xen/xen-head.S          | 12 ++++++++----
 6 files changed, 19 insertions(+), 29 deletions(-)
arch/x86/include/asm/processor.h +5 -8
index 04371f60e3c6..48c31b8e3e72 100644
@@ -393,16 +393,13 @@ struct irq_stack {
 } __aligned(IRQ_STACK_SIZE);
 
 #ifdef CONFIG_X86_64
-struct fixed_percpu_data {
-       char            gs_base[40];
-};
-
-DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
-DECLARE_INIT_PER_CPU(fixed_percpu_data);
-
 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
 {
-       return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
+#ifdef CONFIG_SMP
+       return per_cpu_offset(cpu);
+#else
+       return 0;
+#endif
 }
 
 extern asmlinkage void entry_SYSCALL32_ignore(void);
arch/x86/kernel/cpu/common.c +0 -4
index f9c8bd27b642..a44fd3ad460e 100644
@@ -2051,10 +2051,6 @@ DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
 EXPORT_PER_CPU_SYMBOL(pcpu_hot);
 
 #ifdef CONFIG_X86_64
-DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
-                    fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
-EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
-
 static void wrmsrl_cstar(unsigned long val)
 {
        /*
arch/x86/kernel/head_64.S +6 -6
index 3dcabbc49149..f2453eb38417 100644
@@ -72,9 +72,14 @@ SYM_CODE_START_NOALIGN(startup_64)
 
        /* Setup GSBASE to allow stack canary access for C code */
        movl    $MSR_GS_BASE, %ecx
-       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
+#ifdef CONFIG_SMP
+       leaq    __per_cpu_load(%rip), %rdx
        movl    %edx, %eax
        shrq    $32,  %rdx
+#else
+       xorl    %eax, %eax
+       xorl    %edx, %edx
+#endif
        wrmsr
 
        call    startup_64_setup_env
@@ -345,15 +350,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 
        /* Set up %gs.
         *
-        * The base of %gs always points to fixed_percpu_data. If the
-        * stack protector canary is enabled, it is located at %gs:40.
         * Note that, on SMP, the boot cpu uses init data section until
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-#ifndef CONFIG_SMP
-       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
-#endif
        movl    %edx, %eax
        shrq    $32, %rdx
        wrmsr
arch/x86/kernel/vmlinux.lds.S +0 -6
index 54a5596adaa6..c87dc8de2084 100644
@@ -509,14 +509,8 @@ SECTIONS
  */
 #define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
 INIT_PER_CPU(gdt_page);
-INIT_PER_CPU(fixed_percpu_data);
 INIT_PER_CPU(irq_stack_backing_store);
 
-#ifdef CONFIG_SMP
-. = ASSERT((fixed_percpu_data == 0),
-           "fixed_percpu_data is not at start of per-cpu area");
-#endif
-
 #ifdef CONFIG_CPU_UNRET_ENTRY
 . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
 #endif
arch/x86/tools/relocs.c +0 -1
index d30949e25ebd..3ccd9d4fcf9c 100644
@@ -811,7 +811,6 @@ static void percpu_init(void)
  *     __per_cpu_load
  *
  * The "gold" linker incorrectly associates:
- *     init_per_cpu__fixed_percpu_data
  *     init_per_cpu__gdt_page
  */
 static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
arch/x86/xen/xen-head.S +1 -4
index a0ea285878db..9ce0d9d268bb 100644
@@ -53,14 +53,18 @@ SYM_CODE_START(startup_xen)
 
        /* Set up %gs.
         *
-        * The base of %gs always points to fixed_percpu_data.  If the
-        * stack protector canary is enabled, it is located at %gs:40.
         * Note that, on SMP, the boot cpu uses init data section until
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-       movq    $INIT_PER_CPU_VAR(fixed_percpu_data),%rax
-       cdq
+#ifdef CONFIG_SMP

arch/x86/xen/xen-head.S SYM_CODE_START(startup_xen) · line 70

Quoted 10 lines
arch/x86/xen/xen-head.S +7 -0
+       leaq    __per_cpu_load(%rip), %rdx
+       movl    %edx, %eax
+       shrq    $32, %rdx
+#else
+       xorl    %eax, %eax
+       xorl    %edx, %edx
+#endif
        wrmsr
 
        mov     %rsi, %rdi

Please note there is another access to $MSR_GS_BASE in /arch/x86/platform/pvh/head.S around line 98. Should this be fixed, too?

Uros.

#4 lore raw

Re: [PATCH 3/9] x86/stackprotector/64: Convert stack protector to normal percpu variable

In reply to Brian Gerst
Quoted 33 lines · Brian Gerst

Older versions of GCC fixed the location of the stack protector canary at %gs:40. This constraint forced the percpu section to be linked at virtual address 0 so that the canary could be the first data object in the percpu section. Supporting the zero-based percpu section requires additional code to handle relocations for RIP-relative references to percpu data, extra complexity to kallsyms, and workarounds for linker bugs due to the use of absolute symbols.

Since version 8.1, GCC has options to configure the location of the canary value. This allows the canary to be turned into a normal percpu variable and removes the constraint that the percpu section be zero-based.

---
 arch/x86/Kconfig                          |  5 ++-
 arch/x86/Makefile                         | 19 +++++++-----
 arch/x86/entry/entry_64.S                 |  2 +-
 arch/x86/include/asm/processor.h          | 15 +--------
 arch/x86/include/asm/stackprotector.h     | 37 +++++------------------
 arch/x86/kernel/asm-offsets_64.c          |  6 ----
 arch/x86/kernel/cpu/common.c              |  4 +--
 scripts/gcc-x86_64-has-stack-protector.sh |  4 ---
 8 files changed, 24 insertions(+), 68 deletions(-)
 delete mode 100755 scripts/gcc-x86_64-has-stack-protector.sh
arch/x86/Kconfig +0 -0
index 039872be1630..c95e0ce557da 100644
@@ -410,12 +410,11 @@ config PGTABLE_LEVELS
 

arch/x86/Kconfig config PGTABLE_LEVELS · line 417

Quoted 10 lines
arch/x86/Kconfig +2 -3
 config CC_HAS_SANE_STACKPROTECTOR
        bool
-       default $(success,$(srctree)/scripts/gcc-x86_64-has-stack-protector.sh $(CC) $(CLANG_FLAGS)) if 64BIT
+       default $(cc-option,-mstack-protector-guard-reg=gs -mstack-protector-guard-symbol=__stack_chk_guard) if 64BIT
        default $(cc-option,-mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard)
        help
          We have to make sure stack protector is unconditionally disabled if
-         the compiler produces broken code or if it does not let us control
-         the segment on 32-bit kernels.
+         the compiler does not allow control of the segment and symbol.

All compilers that support -mstack-protector-guard-reg=gs -mstack-protector-guard-symbol=__stack_chk_guard correctly emit segment prefix register, so checking for broken code here is indeed overkill. gcc-x86_64-has-stack-protector.sh can rightfully be removed.

Quoted 24 lines
arch/x86/Kconfig +0 -0
 menu "Processor type and features"
 
arch/x86/Makefile +1 -7
index 22e41d9dbc23..169f086997ec 100644
@@ -111,13 +111,7 @@ ifeq ($(CONFIG_X86_32),y)
         # temporary until string.h is fixed
         KBUILD_CFLAGS += -ffreestanding
 
-       ifeq ($(CONFIG_STACKPROTECTOR),y)
-               ifeq ($(CONFIG_SMP),y)
-                       KBUILD_CFLAGS += -mstack-protector-guard-reg=fs -mstack-protector-guard-symbol=__stack_chk_guard
-               else
-                       KBUILD_CFLAGS += -mstack-protector-guard=global
-               endif
-       endif
+       percpu_seg := fs
 else
         BITS := 64
         UTS_MACHINE := x86_64
@@ -167,6 +161,17 @@ else
         KBUILD_CFLAGS += -mcmodel=kernel

arch/x86/Makefile else · line 171

Quoted 10 lines
arch/x86/Makefile +8 -0
         KBUILD_RUSTFLAGS += -Cno-redzone=y
         KBUILD_RUSTFLAGS += -Ccode-model=kernel
+
+       percpu_seg := gs
+endif
+
+ifeq ($(CONFIG_STACKPROTECTOR),y)
+       ifeq ($(CONFIG_SMP),y)
+               KBUILD_CFLAGS += -mstack-protector-guard-reg=$(percpu_seg) \
+                                -mstack-protector-guard-symbol=__stack_chk_guard

Maybe use here:

KBUILD_CFLAGS += -mstack-protector-guard-reg=$(percpu_seg) KBUILD_CFLAGS += -mstack-protector-guard-symbol=__stack_chk_guard

Looks nicer...

Uros.

Quoted 174 lines
arch/x86/Makefile +3 -0
+       else
+               KBUILD_CFLAGS += -mstack-protector-guard=global
+       endif
 endif
 
 #
arch/x86/entry/entry_64.S +1 -1
index 1a88ad8a7b48..cddcc236aaae 100644
@@ -190,7 +190,7 @@ SYM_FUNC_START(__switch_to_asm)
 
 #ifdef CONFIG_STACKPROTECTOR
        movq    TASK_stack_canary(%rsi), %rbx
-       movq    %rbx, PER_CPU_VAR(fixed_percpu_data + FIXED_stack_canary)
+       movq    %rbx, PER_CPU_VAR(__stack_chk_guard)
 #endif
 
        /*
arch/x86/include/asm/processor.h +1 -14
index ae81a7191c1c..04371f60e3c6 100644
@@ -394,16 +394,7 @@ struct irq_stack {
 
 #ifdef CONFIG_X86_64
 struct fixed_percpu_data {
-       /*
-        * GCC hardcodes the stack canary as %gs:40.  Since the
-        * irq_stack is the object at %gs:0, we reserve the bottom
-        * 48 bytes of the irq stack for the canary.
-        *
-        * Once we are willing to require -mstack-protector-guard-symbol=
-        * support for x86_64 stackprotector, we can get rid of this.
-        */
        char            gs_base[40];
-       unsigned long   stack_canary;
 };
 
 DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
@@ -418,11 +409,7 @@ extern asmlinkage void entry_SYSCALL32_ignore(void);
 
 /* Save actual FS/GS selectors and bases to current->thread */
 void current_save_fsgs(void);
-#else  /* X86_64 */
-#ifdef CONFIG_STACKPROTECTOR
-DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
-#endif
-#endif /* !X86_64 */
+#endif /* X86_64 */
 
 struct perf_event;
 
arch/x86/include/asm/stackprotector.h +7 -30
index 00473a650f51..33abbd29ea26 100644
@@ -2,26 +2,13 @@
 /*
  * GCC stack protector support.
  *
- * Stack protector works by putting predefined pattern at the start of
+ * Stack protector works by putting a predefined pattern at the start of
  * the stack frame and verifying that it hasn't been overwritten when
- * returning from the function.  The pattern is called stack canary
- * and unfortunately gcc historically required it to be at a fixed offset
- * from the percpu segment base.  On x86_64, the offset is 40 bytes.
+ * returning from the function.  The pattern is called the stack canary
+ * and is a unique value for each task.
  *
- * The same segment is shared by percpu area and stack canary.  On
- * x86_64, percpu symbols are zero based and %gs (64-bit) points to the
- * base of percpu area.  The first occupant of the percpu area is always
- * fixed_percpu_data which contains stack_canary at the appropriate
- * offset.  On x86_32, the stack canary is just a regular percpu
- * variable.
- *
- * Putting percpu data in %fs on 32-bit is a minor optimization compared to
- * using %gs.  Since 32-bit userspace normally has %fs == 0, we are likely
- * to load 0 into %fs on exit to usermode, whereas with percpu data in
- * %gs, we are likely to load a non-null %gs on return to user mode.
- *
- * Once we are willing to require GCC 8.1 or better for 64-bit stackprotector
- * support, we can remove some of this complexity.
+ * GCC is configured to read the stack canary value from the __stack_chk_guard
+ * per-cpu variable, which is changed on task switch.
  */
 
 #ifndef _ASM_STACKPROTECTOR_H
@@ -36,6 +23,8 @@
 
 #include <linux/sched.h>
 
+DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
+
 /*
  * Initialize the stackprotector canary value.
  *
@@ -51,25 +40,13 @@ static __always_inline void boot_init_stack_canary(void)
 {
        unsigned long canary = get_random_canary();
 
-#ifdef CONFIG_X86_64
-       BUILD_BUG_ON(offsetof(struct fixed_percpu_data, stack_canary) != 40);
-#endif
-
        current->stack_canary = canary;
-#ifdef CONFIG_X86_64
-       this_cpu_write(fixed_percpu_data.stack_canary, canary);
-#else
        this_cpu_write(__stack_chk_guard, canary);
-#endif
 }
 
 static inline void cpu_init_stack_canary(int cpu, struct task_struct *idle)
 {
-#ifdef CONFIG_X86_64
-       per_cpu(fixed_percpu_data.stack_canary, cpu) = idle->stack_canary;
-#else
        per_cpu(__stack_chk_guard, cpu) = idle->stack_canary;
-#endif
 }
 
 #else  /* STACKPROTECTOR */
arch/x86/kernel/asm-offsets_64.c +0 -6
index bb65371ea9df..590b6cd0eac0 100644
@@ -54,11 +54,5 @@ int main(void)
        BLANK();
 #undef ENTRY
 
-       BLANK();
-
-#ifdef CONFIG_STACKPROTECTOR
-       OFFSET(FIXED_stack_canary, fixed_percpu_data, stack_canary);
-       BLANK();
-#endif
        return 0;
 }
arch/x86/kernel/cpu/common.c +1 -3
index b14fc8c1c953..f9c8bd27b642 100644
@@ -2103,15 +2103,13 @@ void syscall_init(void)
               X86_EFLAGS_AC|X86_EFLAGS_ID);
 }
 
-#else  /* CONFIG_X86_64 */
+#endif /* CONFIG_X86_64 */
 
 #ifdef CONFIG_STACKPROTECTOR
 DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
 #endif
 
-#endif /* CONFIG_X86_64 */
-
 /*
  * Clear all 6 debug registers:
  */
scripts/gcc-x86_64-has-stack-protector.sh deleted +0 -5
deleted file mode 100755
index 75e4e22b986a..000000000000
@@ -1,4 +0,0 @@
-#!/bin/sh
-# SPDX-License-Identifier: GPL-2.0
-
-echo "int foo(void) { char X[200]; return 3; }" | $* -S -x c -c -m64 -O0 -mcmodel=kernel -fno-PIE -fstack-protector - -o - 2> /dev/null | grep -q "%gs"
--

2.41.0

#9 lore raw

Re: [PATCH 5/9] x86/percpu/64: Use relative percpu offsets

In reply to Brian Gerst
Quoted 158 lines · Brian Gerst

The percpu section is currently linked at virtual address 0, because older compilers hardcoded the stack protector canary value at a fixed offset from the start of the GS segment. Now that the canary is a normal percpu variable, the percpu section can be linked normally. This means that x86-64 will calculate percpu offsets like most other architectures, as the delta between the initial percpu address and the dynamically allocated memory.

---
 arch/x86/kernel/head_64.S      |  6 ------
 arch/x86/kernel/setup_percpu.c | 12 ++----------
 arch/x86/kernel/vmlinux.lds.S  | 24 +-----------------------
 arch/x86/tools/relocs.c        | 10 +++-------
 arch/x86/xen/xen-head.S        |  6 ------
 init/Kconfig                   |  2 +-
 6 files changed, 7 insertions(+), 53 deletions(-)
arch/x86/kernel/head_64.S +0 -6
index f2453eb38417..b35f74e58dd7 100644
@@ -72,14 +72,8 @@ SYM_CODE_START_NOALIGN(startup_64)
 
        /* Setup GSBASE to allow stack canary access for C code */
        movl    $MSR_GS_BASE, %ecx
-#ifdef CONFIG_SMP
-       leaq    __per_cpu_load(%rip), %rdx
-       movl    %edx, %eax
-       shrq    $32,  %rdx
-#else
        xorl    %eax, %eax
        xorl    %edx, %edx
-#endif
        wrmsr
 
        call    startup_64_setup_env
arch/x86/kernel/setup_percpu.c +2 -10
index 2c97bf7b56ae..8707dd07b9ce 100644
@@ -23,18 +23,10 @@
 #include <asm/cpumask.h>
 #include <asm/cpu.h>
 
-#ifdef CONFIG_X86_64
-#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
-#else
-#define BOOT_PERCPU_OFFSET 0
-#endif
-
-DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
 EXPORT_PER_CPU_SYMBOL(this_cpu_off);
 
-unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init = {
-       [0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
-};
+unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init;
 EXPORT_SYMBOL(__per_cpu_offset);
 
 /*
arch/x86/kernel/vmlinux.lds.S +1 -23
index c87dc8de2084..8d8eb4d9ff9d 100644
@@ -102,12 +102,6 @@ jiffies = jiffies_64;
 PHDRS {
        text PT_LOAD FLAGS(5);          /* R_E */
        data PT_LOAD FLAGS(6);          /* RW_ */
-#ifdef CONFIG_X86_64
-#ifdef CONFIG_SMP
-       percpu PT_LOAD FLAGS(6);        /* RW_ */
-#endif
-       init PT_LOAD FLAGS(7);          /* RWE */
-#endif
        note PT_NOTE FLAGS(0);          /* ___ */
 }
 
@@ -223,21 +217,7 @@ SECTIONS
                __init_begin = .; /* paired with __init_end */
        }
 
-#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
-       /*
-        * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
-        * output PHDR, so the next output section - .init.text - should
-        * start another segment - init.
-        */
-       PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu)
-       ASSERT(SIZEOF(.data..percpu) < CONFIG_PHYSICAL_START,
-              "per-CPU data too large - increase CONFIG_PHYSICAL_START")
-#endif
-
        INIT_TEXT_SECTION(PAGE_SIZE)
-#ifdef CONFIG_X86_64
-       :init
-#endif
 
        /*
         * Section for code used exclusively before alternatives are run. All
@@ -367,9 +347,7 @@ SECTIONS
                EXIT_DATA
        }
 
-#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
        PERCPU_SECTION(INTERNODE_CACHE_BYTES)
-#endif
 
        . = ALIGN(PAGE_SIZE);
 
@@ -507,7 +485,7 @@ SECTIONS
  * Per-cpu symbols which need to be offset from __per_cpu_load
  * for the boot processor.
  */
-#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
+#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x)
 INIT_PER_CPU(gdt_page);
 INIT_PER_CPU(irq_stack_backing_store);
 
arch/x86/tools/relocs.c +3 -7
index 3ccd9d4fcf9c..01efbfdd3eb3 100644
@@ -815,12 +815,7 @@ static void percpu_init(void)
  */
 static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
 {
-       int shndx = sym_index(sym);
-
-       return (shndx == per_cpu_shndx) &&
-               strcmp(symname, "__init_begin") &&
-               strcmp(symname, "__per_cpu_load") &&
-               strncmp(symname, "init_per_cpu_", 13);
+       return 0;
 }
 
 
@@ -1043,7 +1038,8 @@ static int cmp_relocs(const void *va, const void *vb)
 
 static void sort_relocs(struct relocs *r)
 {
-       qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
+       if (r->count)
+               qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
 }
 
 static int write32(uint32_t v, FILE *f)
arch/x86/xen/xen-head.S +0 -1
index 9ce0d9d268bb..c1d9c92b417a 100644
@@ -57,14 +57,8 @@ SYM_CODE_START(startup_xen)
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-#ifdef CONFIG_SMP

arch/x86/xen/xen-head.S SYM_CODE_START(startup_xen) · line 64

Quoted 10 lines
arch/x86/xen/xen-head.S +0 -5
-       leaq    __per_cpu_load(%rip), %rdx
-       movl    %edx, %eax
-       shrq    $32, %rdx
-#else
        xorl    %eax, %eax
        xorl    %edx, %edx
-#endif
        wrmsr
 
        mov     %rsi, %rdi

Again, please note there is another access to $MSR_GS_BASE in /arch/x86/platform/pvh/head.S that should be reviewed.

Uros.

Quoted 16 lines
init/Kconfig +1 -2
index 6d35728b94b2..1af31b23e376 100644
@@ -1718,7 +1718,7 @@ config KALLSYMS_ALL
 config KALLSYMS_ABSOLUTE_PERCPU
        bool
        depends on KALLSYMS
-       default X86_64 && SMP
+       default n
 
 config KALLSYMS_BASE_RELATIVE
        bool
--

2.41.0

#7 lore raw

Re: [PATCH 4/9] x86/percpu/64: Remove fixed_percpu_data

In reply to Uros Bizjak
Quoted 150 lines · Uros Bizjak
Quoted 148 lines · Brian Gerst

Now that the stack protector canary value is a normal percpu variable, fixed_percpu_data is unused and can be removed.

---
 arch/x86/include/asm/processor.h | 13 +++++--------
 arch/x86/kernel/cpu/common.c     |  4 ----
 arch/x86/kernel/head_64.S        | 12 ++++++------
 arch/x86/kernel/vmlinux.lds.S    |  6 ------
 arch/x86/tools/relocs.c          |  1 -
 arch/x86/xen/xen-head.S          | 12 ++++++++----
 6 files changed, 19 insertions(+), 29 deletions(-)
arch/x86/include/asm/processor.h +5 -8
index 04371f60e3c6..48c31b8e3e72 100644
@@ -393,16 +393,13 @@ struct irq_stack {
 } __aligned(IRQ_STACK_SIZE);
 
 #ifdef CONFIG_X86_64
-struct fixed_percpu_data {
-       char            gs_base[40];
-};
-
-DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
-DECLARE_INIT_PER_CPU(fixed_percpu_data);
-
 static inline unsigned long cpu_kernelmode_gs_base(int cpu)
 {
-       return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
+#ifdef CONFIG_SMP
+       return per_cpu_offset(cpu);
+#else
+       return 0;
+#endif
 }
 
 extern asmlinkage void entry_SYSCALL32_ignore(void);
arch/x86/kernel/cpu/common.c +0 -4
index f9c8bd27b642..a44fd3ad460e 100644
@@ -2051,10 +2051,6 @@ DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
 EXPORT_PER_CPU_SYMBOL(pcpu_hot);
 
 #ifdef CONFIG_X86_64
-DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
-                    fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
-EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
-
 static void wrmsrl_cstar(unsigned long val)
 {
        /*
arch/x86/kernel/head_64.S +6 -6
index 3dcabbc49149..f2453eb38417 100644
@@ -72,9 +72,14 @@ SYM_CODE_START_NOALIGN(startup_64)
 
        /* Setup GSBASE to allow stack canary access for C code */
        movl    $MSR_GS_BASE, %ecx
-       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
+#ifdef CONFIG_SMP
+       leaq    __per_cpu_load(%rip), %rdx
        movl    %edx, %eax
        shrq    $32,  %rdx
+#else
+       xorl    %eax, %eax
+       xorl    %edx, %edx
+#endif
        wrmsr
 
        call    startup_64_setup_env
@@ -345,15 +350,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 
        /* Set up %gs.
         *
-        * The base of %gs always points to fixed_percpu_data. If the
-        * stack protector canary is enabled, it is located at %gs:40.
         * Note that, on SMP, the boot cpu uses init data section until
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-#ifndef CONFIG_SMP
-       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
-#endif
        movl    %edx, %eax
        shrq    $32, %rdx
        wrmsr
arch/x86/kernel/vmlinux.lds.S +0 -6
index 54a5596adaa6..c87dc8de2084 100644
@@ -509,14 +509,8 @@ SECTIONS
  */
 #define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
 INIT_PER_CPU(gdt_page);
-INIT_PER_CPU(fixed_percpu_data);
 INIT_PER_CPU(irq_stack_backing_store);
 
-#ifdef CONFIG_SMP
-. = ASSERT((fixed_percpu_data == 0),
-           "fixed_percpu_data is not at start of per-cpu area");
-#endif
-
 #ifdef CONFIG_CPU_UNRET_ENTRY
 . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
 #endif
arch/x86/tools/relocs.c +0 -1
index d30949e25ebd..3ccd9d4fcf9c 100644
@@ -811,7 +811,6 @@ static void percpu_init(void)
  *     __per_cpu_load
  *
  * The "gold" linker incorrectly associates:
- *     init_per_cpu__fixed_percpu_data
  *     init_per_cpu__gdt_page
  */
 static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
arch/x86/xen/xen-head.S +8 -4
index a0ea285878db..9ce0d9d268bb 100644
@@ -53,14 +53,18 @@ SYM_CODE_START(startup_xen)
 
        /* Set up %gs.
         *
-        * The base of %gs always points to fixed_percpu_data.  If the
-        * stack protector canary is enabled, it is located at %gs:40.
         * Note that, on SMP, the boot cpu uses init data section until
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-       movq    $INIT_PER_CPU_VAR(fixed_percpu_data),%rax
-       cdq
+#ifdef CONFIG_SMP
+       leaq    __per_cpu_load(%rip), %rdx
+       movl    %edx, %eax
+       shrq    $32, %rdx
+#else
+       xorl    %eax, %eax
+       xorl    %edx, %edx
+#endif
        wrmsr
 
Quoted 5 lines

arch/x86/xen/xen-head.S SYM_CODE_START(startup_xen) · line 70

Quoted 1 line
arch/x86/xen/xen-head.S +0 -0
        mov     %rsi, %rdi

Please note there is another access to $MSR_GS_BASE in /arch/x86/platform/pvh/head.S around line 98. Should this be fixed, too?

Thanks for pointing that out. I missed it since it didn't reference fixed_percpu_data. Fixed for v2.

Brian Gerst