x86: Kernel IBT
Peter Zijlstra
maintainerv4 cover x86: Kernel IBT
Hopefully last posting...
Since last time:
- updated the ftrace_location() patch (naveen, rostedt)
- added a few comments and clarifications (bpetkov)
- disable jump-tables (joao)
- verified clang-14-rc2 works
- fixed a whole bunch of objtool unreachable insn issue
- picked up a few more tags
Patches go on top of tip/master + arm64/for-next/linkage. Also available here:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/wip.ibt
Enjoy!
Peter Zijlstra
maintainerPATCH v4 1/45 static_call: Avoid building empty .static_call_sites
Without CONFIG_HAVE_STATIC_CALL_INLINE there's no point in creating the .static_call_sites section and it's related symbols.
--- include/asm-generic/vmlinux.lds.h | 4 ++++ 1 file changed, 4 insertions(+)
include/asm-generic/vmlinux.lds.h +4 -0
@@ -398,6 +398,7 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE #define STATIC_CALL_DATA \ . = ALIGN(8); \ __start_static_call_sites = .; \
@@ -406,6 +407,9 @@ __start_static_call_tramp_key = .; \ KEEP(*(.static_call_tramp_key)) \ __stop_static_call_tramp_key = .; +#else +#define STATIC_CALL_DATA +#endif /* * Allow architectures to handle ro_after_init data on their
[tip: x86/core] static_call: Avoid building empty .static_call_sites
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 01ddabda5a3e16eace6d61bcf1857eba8f015b13 Gitweb: https://git.kernel.org/tip/01ddabda5a3e16eace6d61bcf1857eba8f015b13 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:12 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:26 +01:00
static_call: Avoid building empty .static_call_sites
Without CONFIG_HAVE_STATIC_CALL_INLINE there's no point in creating the .static_call_sites section and it's related symbols.
--- include/asm-generic/vmlinux.lds.h | 4 ++++ 1 file changed, 4 insertions(+)
include/asm-generic/vmlinux.lds.h +4 -0
@@ -404,6 +404,7 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE #define STATIC_CALL_DATA \ . = ALIGN(8); \ __start_static_call_sites = .; \
@@ -412,6 +413,9 @@ __start_static_call_tramp_key = .; \ KEEP(*(.static_call_tramp_key)) \ __stop_static_call_tramp_key = .; +#else +#define STATIC_CALL_DATA +#endif /* * Allow architectures to handle ro_after_init data on their
[tip: x86/core] static_call: Avoid building empty .static_call_sites
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: b44544fe0298ee2224960a31f795e317029e2a60 Gitweb: https://git.kernel.org/tip/b44544fe0298ee2224960a31f795e317029e2a60 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:12 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:31 +01:00
static_call: Avoid building empty .static_call_sites
Without CONFIG_HAVE_STATIC_CALL_INLINE there's no point in creating the .static_call_sites section and it's related symbols.
--- include/asm-generic/vmlinux.lds.h | 4 ++++ 1 file changed, 4 insertions(+)
include/asm-generic/vmlinux.lds.h +4 -0
@@ -404,6 +404,7 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE #define STATIC_CALL_DATA \ . = ALIGN(8); \ __start_static_call_sites = .; \
@@ -412,6 +413,9 @@ __start_static_call_tramp_key = .; \ KEEP(*(.static_call_tramp_key)) \ __stop_static_call_tramp_key = .; +#else +#define STATIC_CALL_DATA +#endif /* * Allow architectures to handle ro_after_init data on their
Peter Zijlstra
maintainerPATCH v4 2/45 objtool: Add --dry-run
In reply to Peter ZijlstraAdd a --dry-run argument to skip writing the modifications. This is convenient for debugging.
--- tools/objtool/builtin-check.c | 3 ++- tools/objtool/elf.c | 3 +++ tools/objtool/include/objtool/builtin.h | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-)
tools/objtool/builtin-check.c +2 -1
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -46,6 +46,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('M' ,"mcount" , &mcount,"generate __mcount_loc" ),OPT_BOOLEAN ('B' ,"backup" , &backup,"create .orig files before modification" ),OPT_BOOLEAN ('S' ,"sls" , &sls,"validate straight-line-speculation" ), +OPT_BOOLEAN (0 ,"dry-run" , &dryrun,"don't write the modifications" ), OPT_END(), };
tools/objtool/elf.c +3 -0
@@ -1019,6 +1019,9 @@ int elf_write(struct elf *elf)struct section *sec;Elf_Scn *s; +if (dryrun) +return 0 ; +/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) {
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
[tip: x86/core] objtool: Add --dry-run
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 951c6aeb3bda02073895ace47630f4796a8b6a9d Gitweb: https://git.kernel.org/tip/951c6aeb3bda02073895ace47630f4796a8b6a9d Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:13 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:27 +01:00
objtool: Add --dry-run
Add a --dry-run argument to skip writing the modifications. This is convenient for debugging.
--- tools/objtool/builtin-check.c | 3 ++- tools/objtool/elf.c | 3 +++ tools/objtool/include/objtool/builtin.h | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-)
tools/objtool/builtin-check.c +2 -1
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -46,6 +46,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('M' ,"mcount" , &mcount,"generate __mcount_loc" ),OPT_BOOLEAN ('B' ,"backup" , &backup,"create .orig files before modification" ),OPT_BOOLEAN ('S' ,"sls" , &sls,"validate straight-line-speculation" ), +OPT_BOOLEAN (0 ,"dry-run" , &dryrun,"don't write the modifications" ), OPT_END(), };
tools/objtool/elf.c +3 -0
@@ -1019,6 +1019,9 @@ int elf_write(struct elf *elf)struct section *sec;Elf_Scn *s; +if (dryrun) +return 0 ; +/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) {
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
[tip: x86/core] objtool: Add --dry-run
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f2d3a250897133cc36c13a641bd6a9b4dd5ad234 Gitweb: https://git.kernel.org/tip/f2d3a250897133cc36c13a641bd6a9b4dd5ad234 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:13 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:32 +01:00
objtool: Add --dry-run
Add a --dry-run argument to skip writing the modifications. This is convenient for debugging.
--- tools/objtool/builtin-check.c | 3 ++- tools/objtool/elf.c | 3 +++ tools/objtool/include/objtool/builtin.h | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-)
tools/objtool/builtin-check.c +2 -1
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -46,6 +46,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('M' ,"mcount" , &mcount,"generate __mcount_loc" ),OPT_BOOLEAN ('B' ,"backup" , &backup,"create .orig files before modification" ),OPT_BOOLEAN ('S' ,"sls" , &sls,"validate straight-line-speculation" ), +OPT_BOOLEAN (0 ,"dry-run" , &dryrun,"don't write the modifications" ), OPT_END(), };
tools/objtool/elf.c +3 -0
@@ -1019,6 +1019,9 @@ int elf_write(struct elf *elf)struct section *sec;Elf_Scn *s; +if (dryrun) +return 0 ; +/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) {
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls; + validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
Peter Zijlstra
maintainerPATCH v4 3/45 objtool: Default ignore INT3 for unreachable
In reply to Peter ZijlstraIgnore all INT3 instructions for unreachable code warnings, similar to NOP. This allows using INT3 for various paddings instead of NOPs.
--- tools/objtool/check.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
tools/objtool/check.c +5 -7
@@ -3342,9 +3342,8 @@ static int validate_branch(struct objtooswitch (insn->type) {case INSN_RETURN : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after ret" , insn->sec, insn->offset); }
@@ -3391,9 +3390,8 @@ static int validate_branch(struct objtoobreak ;case INSN_JUMP_DYNAMIC : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after indirect jump" , insn->sec, insn->offset); }
@@ -3567,7 +3565,7 @@ static bool ignore_unreachable_insn(struint i;struct instruction *prev_insn; -if (insn->ignore || insn->type ==INSN_NOP ) +if (insn->ignore || insn->type ==INSN_NOP || insn->type ==INSN_TRAP )return true;/*
[tip: x86/core] objtool: Default ignore INT3 for unreachable
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: def466688a10fe8a2e1da7d3a72b4a29ff6b1a9e Gitweb: https://git.kernel.org/tip/def466688a10fe8a2e1da7d3a72b4a29ff6b1a9e Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:14 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:27 +01:00
objtool: Default ignore INT3 for unreachable
Ignore all INT3 instructions for unreachable code warnings, similar to NOP. This allows using INT3 for various paddings instead of NOPs.
--- tools/objtool/check.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
tools/objtool/check.c +5 -7
@@ -3115,9 +3115,8 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,switch (insn->type) {case INSN_RETURN : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after ret" , insn->sec, insn->offset); }
@@ -3164,9 +3163,8 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,break ;case INSN_JUMP_DYNAMIC : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after indirect jump" , insn->sec, insn->offset); }
@@ -3337,7 +3335,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructioint i;struct instruction *prev_insn; -if (insn->ignore || insn->type ==INSN_NOP ) +if (insn->ignore || insn->type ==INSN_NOP || insn->type ==INSN_TRAP )return true;/*
[tip: x86/core] objtool: Default ignore INT3 for unreachable
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 1ffbe4e935f9b7308615c75be990aec07464d1e7 Gitweb: https://git.kernel.org/tip/1ffbe4e935f9b7308615c75be990aec07464d1e7 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:14 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:32 +01:00
objtool: Default ignore INT3 for unreachable
Ignore all INT3 instructions for unreachable code warnings, similar to NOP. This allows using INT3 for various paddings instead of NOPs.
--- tools/objtool/check.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
tools/objtool/check.c +5 -7
@@ -3115,9 +3115,8 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,switch (insn->type) {case INSN_RETURN : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after ret" , insn->sec, insn->offset); }
@@ -3164,9 +3163,8 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,break ;case INSN_JUMP_DYNAMIC : -if (next_insn && next_insn->type ==INSN_TRAP ) { - next_insn->ignore = true; - }else if (sls && !insn->retpoline_safe) { +if (sls && !insn->retpoline_safe && + next_insn && next_insn->type !=INSN_TRAP ) {WARN_FUNC ("missing int3 after indirect jump" , insn->sec, insn->offset); }
@@ -3337,7 +3335,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructioint i;struct instruction *prev_insn; -if (insn->ignore || insn->type ==INSN_NOP ) +if (insn->ignore || insn->type ==INSN_NOP || insn->type ==INSN_TRAP )return true;/*
Peter Zijlstra
maintainerPATCH v4 4/45 objtool,efi: Update __efi64_thunk annotation
In reply to Peter ZijlstraThe current annotation relies on not running objtool on the file; this won't work when running objtool on vmlinux.o. Instead explicitly mark __efi64_thunk() to be ignored.
This preserves the status quo, which is somewhat unfortunate. Luckily this code is hardly ever used.
--- arch/x86/platform/efi/Makefile | 1 - arch/x86/platform/efi/efi_thunk_64.S | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-)
arch/x86/platform/efi/Makefile +0 -1
@@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_efi_thunk_$(BITS).o := y KASAN_SANITIZE := n GCOV_PROFILE := n
arch/x86/platform/efi/efi_thunk_64.S +4 -2
@@ -20,12 +20,14 @@ */ #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/segment.h> .text .code64 -SYM_CODE_START(__efi64_thunk) +SYM_FUNC_START(__efi64_thunk) +STACK_FRAME_NON_STANDARD __efi64_thunk push %rbp push %rbx
@@ -79,7 +81,7 @@ SYM_CODE_START(__efi64_thunk) 2: pushl $__KERNEL_CS pushl %ebp lret -SYM_CODE_END(__efi64_thunk) +SYM_FUNC_END(__efi64_thunk) .bss .balign 8
[tip: x86/core] objtool,efi: Update __efi64_thunk annotation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f99b84d14feba2ac011b1a21ceae4aa9e3fe49e8 Gitweb: https://git.kernel.org/tip/f99b84d14feba2ac011b1a21ceae4aa9e3fe49e8 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:15 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:27 +01:00
objtool,efi: Update __efi64_thunk annotation
The current annotation relies on not running objtool on the file; this won't work when running objtool on vmlinux.o. Instead explicitly mark __efi64_thunk() to be ignored.
This preserves the status quo, which is somewhat unfortunate. Luckily this code is hardly ever used.
--- arch/x86/platform/efi/Makefile | 1 - arch/x86/platform/efi/efi_thunk_64.S | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-)
arch/x86/platform/efi/Makefile +0 -1
@@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_efi_thunk_$(BITS).o := y KASAN_SANITIZE := n GCOV_PROFILE := n
arch/x86/platform/efi/efi_thunk_64.S +4 -2
@@ -20,12 +20,14 @@ */ #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/segment.h> .text .code64 -SYM_CODE_START(__efi64_thunk) +SYM_FUNC_START(__efi64_thunk) +STACK_FRAME_NON_STANDARD __efi64_thunk push %rbp push %rbx
@@ -79,7 +81,7 @@ SYM_CODE_START(__efi64_thunk) 2: pushl $__KERNEL_CS pushl %ebp lret -SYM_CODE_END(__efi64_thunk) +SYM_FUNC_END(__efi64_thunk) .bss .balign 8
Miroslav Benes
maintainerRe: [PATCH v4 04/45] objtool,efi: Update __efi64_thunk annotation
In reply to Peter ZijlstraQuoted 5 lines
arch/x86/platform/efi/efi_thunk_64.S +0
-0@@ -20,12 +20,14 @@ */
arch/x86/platform/efi/efi_thunk_64.S line 30
Quoted 10 lines
arch/x86/platform/efi/efi_thunk_64.S +3
-1#include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/segment.h> .text .code64 -SYM_CODE_START(__efi64_thunk) +SYM_FUNC_START(__efi64_thunk) +STACK_FRAME_NON_STANDARD __efi64_thunk
I would prefer if STACK_FRAME_NON_STANDARD annotation...
arch/x86/platform/efi/efi_thunk_64.S SYM_CODE_START(__efi64_thunk) · line 84
Quoted 9 lines
arch/x86/platform/efi/efi_thunk_64.S +1
-1push %rbp push %rbx@@ -79,7 +81,7 @@ SYM_CODE_START(__efi64_thunk) 2: pushl $__KERNEL_CS pushl %ebp lret -SYM_CODE_END(__efi64_thunk) +SYM_FUNC_END(__efi64_thunk)
went here after SYM_FUNC_END(), but that is just a cosmetic nit.
Miroslav
[tip: x86/core] objtool,efi: Update __efi64_thunk annotation
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 537da1ed54658e916141e50923a7f5b20c728856 Gitweb: https://git.kernel.org/tip/537da1ed54658e916141e50923a7f5b20c728856 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:15 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:32 +01:00
objtool,efi: Update __efi64_thunk annotation
The current annotation relies on not running objtool on the file; this won't work when running objtool on vmlinux.o. Instead explicitly mark __efi64_thunk() to be ignored.
This preserves the status quo, which is somewhat unfortunate. Luckily this code is hardly ever used.
--- arch/x86/platform/efi/Makefile | 1 - arch/x86/platform/efi/efi_thunk_64.S | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-)
arch/x86/platform/efi/Makefile +0 -1
@@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_efi_thunk_$(BITS).o := y KASAN_SANITIZE := n GCOV_PROFILE := n
arch/x86/platform/efi/efi_thunk_64.S +4 -2
@@ -20,12 +20,14 @@ */ #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/segment.h> .text .code64 -SYM_CODE_START(__efi64_thunk) +SYM_FUNC_START(__efi64_thunk) +STACK_FRAME_NON_STANDARD __efi64_thunk push %rbp push %rbx
@@ -79,7 +81,7 @@ SYM_CODE_START(__efi64_thunk) 2: pushl $__KERNEL_CS pushl %ebp lret -SYM_CODE_END(__efi64_thunk) +SYM_FUNC_END(__efi64_thunk) .bss .balign 8
Peter Zijlstra
maintainerPATCH v4 5/45 objtool: Have WARN_FUNC fall back to sym+off
In reply to Peter ZijlstraCurrently WARN_FUNC() either prints func+off and failing that prints sec+off, add an intermediate sym+off. This is useful when playing around with entry code.
--- tools/objtool/include/objtool/warn.h | 2 ++ 1 file changed, 2 insertions(+)
tools/objtool/include/objtool/warn.h +2 -0
@@ -22,6 +22,8 @@ static inline char *offstr(struct sectiounsigned long name_off; func =find_func_containing (sec, offset); +if (!func) + func =find_symbol_containing (sec, offset);if (func) { name = func->name; name_off = offset - func->offset;
[tip: x86/core] objtool: Have WARN_FUNC fall back to sym+off
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 03232d0842453f256c278c377f9646e45eb57d98 Gitweb: https://git.kernel.org/tip/03232d0842453f256c278c377f9646e45eb57d98 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:16 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:27 +01:00
objtool: Have WARN_FUNC fall back to sym+off
Currently WARN_FUNC() either prints func+off and failing that prints sec+off, add an intermediate sym+off. This is useful when playing around with entry code.
--- tools/objtool/include/objtool/warn.h | 2 ++ 1 file changed, 2 insertions(+)
tools/objtool/include/objtool/warn.h +2 -0
@@ -22,6 +22,8 @@ static inline char *offstr(struct section *sec, unsigned long offset)unsigned long name_off; func =find_func_containing (sec, offset); +if (!func) + func =find_symbol_containing (sec, offset);if (func) { name = func->name; name_off = offset - func->offset;
[tip: x86/core] objtool: Have WARN_FUNC fall back to sym+off
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 5cff2086b01526b8c7deacc86473ffbab0cddfa9 Gitweb: https://git.kernel.org/tip/5cff2086b01526b8c7deacc86473ffbab0cddfa9 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:16 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:32 +01:00
objtool: Have WARN_FUNC fall back to sym+off
Currently WARN_FUNC() either prints func+off and failing that prints sec+off, add an intermediate sym+off. This is useful when playing around with entry code.
--- tools/objtool/include/objtool/warn.h | 2 ++ 1 file changed, 2 insertions(+)
tools/objtool/include/objtool/warn.h +2 -0
@@ -22,6 +22,8 @@ static inline char *offstr(struct section *sec, unsigned long offset)unsigned long name_off; func =find_func_containing (sec, offset); +if (!func) + func =find_symbol_containing (sec, offset);if (func) { name = func->name; name_off = offset - func->offset;
Peter Zijlstra
maintainerPATCH v4 6/45 x86/ibt: Base IBT bits
In reply to Peter ZijlstraAdd Kconfig, Makefile and basic instruction support for x86 IBT.
(Ab)use __DISABLE_EXPORTS to disable IBT since it's already employed to mark compressed and purgatory. Additionally mark realmode with it as well to avoid inserting ENDBR instructions there. While ENDBR is technically a NOP, inserting them was causing some grief due to code growth. There's also a problem with using __noendbr in code compiled without -fcf-protection=branch.
--- arch/x86/Kconfig | 20 ++++++++++ arch/x86/Makefile | 16 +++++++- arch/x86/include/asm/ibt.h | 87 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-)
arch/x86/Kconfig +20 -0
@@ -1861,6 +1861,26 @@ config X86_UMIP specific cases in protected and virtual-8086 modes. Emulated results are dummy. +config CC_HAS_IBT + # GCC >= 9 and binutils >= 2.29 + # Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654 + # Clang/LLVM >= 14 + # fentry check to work around https://reviews.llvm.org/D111108 + def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) || \ + (CC_IS_CLANG && $(cc-option, -fcf-protection=branch -mfentry))) && \ + $(as-instr,endbr64) + +config X86_KERNEL_IBT + prompt "Indirect Branch Tracking" + bool + depends on X86_64 && CC_HAS_IBT + help + Build the kernel with support for Indirect Branch Tracking, a + hardware support course-grain forward-edge Control Flow Integrity + protection. It enforces that all indirect calls must land on + an ENDBR instruction, as such, the compiler will instrument the + code with them to make this happen. + config X86_INTEL_MEMORY_PROTECTION_KEYS prompt "Memory Protection Keys" def_bool y
arch/x86/Makefile +14 -2
@@ -36,7 +36,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \ +REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
@@ -62,8 +62,20 @@ export BITS # KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -# Intel CET isn't enabled in the kernel +ifeq ($(CONFIG_X86_KERNEL_IBT),y) +# +# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate +# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK +# for jump-tables, as such, disable jump-tables for now. +# +# (jump-tables are implicitly disabled by RETPOLINE) +# +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816 +# +KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables) +else KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none) +endif ifeq ($(CONFIG_X86_32),y) BITS := 32
arch/x86/include/asm/ibt.h new file +87 -0
87 lines not on this page · raw message
[tip: x86/core] x86/ibt: Base IBT bits
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 41c5ef31ad7173c9f8469d0148666736e104fbc4 Gitweb: https://git.kernel.org/tip/41c5ef31ad7173c9f8469d0148666736e104fbc4 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:17 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:28 +01:00
x86/ibt: Base IBT bits
Add Kconfig, Makefile and basic instruction support for x86 IBT.
(Ab)use __DISABLE_EXPORTS to disable IBT since it's already employed to mark compressed and purgatory. Additionally mark realmode with it as well to avoid inserting ENDBR instructions there. While ENDBR is technically a NOP, inserting them was causing some grief due to code growth. There's also a problem with using __noendbr in code compiled without -fcf-protection=branch.
--- arch/x86/Kconfig | 20 +++++++++- arch/x86/Makefile | 16 ++++++- arch/x86/include/asm/ibt.h | 87 +++++++++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 arch/x86/include/asm/ibt.h
arch/x86/Kconfig +20 -0
@@ -1861,6 +1861,26 @@ config X86_UMIP specific cases in protected and virtual-8086 modes. Emulated results are dummy. +config CC_HAS_IBT + # GCC >= 9 and binutils >= 2.29 + # Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654 + # Clang/LLVM >= 14 + # fentry check to work around https://reviews.llvm.org/D111108 + def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) || \ + (CC_IS_CLANG && $(cc-option, -fcf-protection=branch -mfentry))) && \ + $(as-instr,endbr64) + +config X86_KERNEL_IBT + prompt "Indirect Branch Tracking" + bool + depends on X86_64 && CC_HAS_IBT + help + Build the kernel with support for Indirect Branch Tracking, a + hardware support course-grain forward-edge Control Flow Integrity + protection. It enforces that all indirect calls must land on + an ENDBR instruction, as such, the compiler will instrument the + code with them to make this happen. + config X86_INTEL_MEMORY_PROTECTION_KEYS prompt "Memory Protection Keys" def_bool y
arch/x86/Makefile +14 -2
@@ -36,7 +36,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \ +REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
@@ -62,8 +62,20 @@ export BITS # KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -# Intel CET isn't enabled in the kernel +ifeq ($(CONFIG_X86_KERNEL_IBT),y) +# +# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate +# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK +# for jump-tables, as such, disable jump-tables for now. +# +# (jump-tables are implicitly disabled by RETPOLINE) +# +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816 +# +KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables) +else KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none) +endif ifeq ($(CONFIG_X86_32),y) BITS := 32
arch/x86/include/asm/ibt.h new file +87 -0
87 lines not on this page · raw message
[tip: x86/core] x86/ibt: Base IBT bits
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 156ff4a544ae13c5fd6759a09ecb069f7059c1a1 Gitweb: https://git.kernel.org/tip/156ff4a544ae13c5fd6759a09ecb069f7059c1a1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:17 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:33 +01:00
x86/ibt: Base IBT bits
Add Kconfig, Makefile and basic instruction support for x86 IBT.
(Ab)use __DISABLE_EXPORTS to disable IBT since it's already employed to mark compressed and purgatory. Additionally mark realmode with it as well to avoid inserting ENDBR instructions there. While ENDBR is technically a NOP, inserting them was causing some grief due to code growth. There's also a problem with using __noendbr in code compiled without -fcf-protection=branch.
--- arch/x86/Kconfig | 20 +++++++++- arch/x86/Makefile | 16 ++++++- arch/x86/include/asm/ibt.h | 87 +++++++++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 arch/x86/include/asm/ibt.h
arch/x86/Kconfig +20 -0
@@ -1861,6 +1861,26 @@ config X86_UMIP specific cases in protected and virtual-8086 modes. Emulated results are dummy. +config CC_HAS_IBT + # GCC >= 9 and binutils >= 2.29 + # Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654 + # Clang/LLVM >= 14 + # fentry check to work around https://reviews.llvm.org/D111108 + def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) || \ + (CC_IS_CLANG && $(success,echo "void a(void) {}" | $(CC) -Werror $(CLANG_FLAGS) -fcf-protection=branch -mfentry -pg -x c - -c -o /dev/null))) && \ + $(as-instr,endbr64) + +config X86_KERNEL_IBT + prompt "Indirect Branch Tracking" + bool + depends on X86_64 && CC_HAS_IBT + help + Build the kernel with support for Indirect Branch Tracking, a + hardware support course-grain forward-edge Control Flow Integrity + protection. It enforces that all indirect calls must land on + an ENDBR instruction, as such, the compiler will instrument the + code with them to make this happen. + config X86_INTEL_MEMORY_PROTECTION_KEYS prompt "Memory Protection Keys" def_bool y
arch/x86/Makefile +14 -2
@@ -36,7 +36,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \ +REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
@@ -62,8 +62,20 @@ export BITS # KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -# Intel CET isn't enabled in the kernel +ifeq ($(CONFIG_X86_KERNEL_IBT),y) +# +# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate +# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK +# for jump-tables, as such, disable jump-tables for now. +# +# (jump-tables are implicitly disabled by RETPOLINE) +# +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816 +# +KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables) +else KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none) +endif ifeq ($(CONFIG_X86_32),y) BITS := 32
arch/x86/include/asm/ibt.h new file +87 -0
87 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v4 7/45 x86/ibt: Add ANNOTATE_NOENDBR
In reply to Peter ZijlstraIn order to have objtool warn about code references to !ENDBR instruction, we need an annotation to allow this for non-control-flow instances -- consider text range checks, text patching, or return trampolines etc.
--- include/linux/objtool.h | 16 ++++++++++++++++ tools/include/linux/objtool.h | 16 ++++++++++++++++ 2 files changed, 32 insertions(+)
include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
tools/include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
[tip: x86/core] x86/ibt: Add ANNOTATE_NOENDBR
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 34963d5e9be5911d93d4a543aa1d07e719599371 Gitweb: https://git.kernel.org/tip/34963d5e9be5911d93d4a543aa1d07e719599371 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:18 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:28 +01:00
x86/ibt: Add ANNOTATE_NOENDBR
In order to have objtool warn about code references to !ENDBR instruction, we need an annotation to allow this for non-control-flow instances -- consider text range checks, text patching, or return trampolines etc.
--- include/linux/objtool.h | 16 ++++++++++++++++ tools/include/linux/objtool.h | 16 ++++++++++++++++ 2 files changed, 32 insertions(+)
include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
tools/include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
[tip: x86/core] x86/ibt: Add ANNOTATE_NOENDBR
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c8c301abeae58ec756b8fcb2178a632bd3c9e284 Gitweb: https://git.kernel.org/tip/c8c301abeae58ec756b8fcb2178a632bd3c9e284 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:18 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:33 +01:00
x86/ibt: Add ANNOTATE_NOENDBR
In order to have objtool warn about code references to !ENDBR instruction, we need an annotation to allow this for non-control-flow instances -- consider text range checks, text patching, or return trampolines etc.
--- include/linux/objtool.h | 16 ++++++++++++++++ tools/include/linux/objtool.h | 16 ++++++++++++++++ 2 files changed, 32 insertions(+)
include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
tools/include/linux/objtool.h +16 -0
@@ -77,6 +77,12 @@ struct unwind_hint {#define STACK_FRAME_NON_STANDARD_FP (func)#endif +#define ANNOTATE_NOENDBR \ + "986: \n\t" \ + ".pushsection .discard.noendbr\n\t" \ + _ASM_PTR " 986b\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -129,6 +135,13 @@ struct unwind_hint { .popsection .endm +.macro ANNOTATE_NOENDBR +.Lhere_\@: + .pushsection .discard.noendbr + .quad .Lhere_\@ + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +152,15 @@ struct unwind_hint { "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) +#define ANNOTATE_NOENDBR #else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm +.macro ANNOTATE_NOENDBR +.endm#endif #endif /* CONFIG_STACK_VALIDATION */
Peter Zijlstra
maintainerPATCH v4 8/45 x86/text-patching: Make text_gen_insn() play nice with ANNOTATE_NOENDBR
In reply to Peter Zijlstra--- arch/x86/include/asm/text-patching.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
arch/x86/include/asm/text-patching.h +9 -1
@@ -101,13 +101,21 @@ void *text_gen_insn(u8 opcode, const voistatic union text_poke_insn insn;/* per instance */ int size =text_opcode_size (opcode); +/* +* Hide the addresses to avoid the compiler folding in constants when +* referencing code, these can mess up annotations like +* ANNOTATE_NOENDBR. +*/ +OPTIMIZER_HIDE_VAR (addr); +OPTIMIZER_HIDE_VAR (dest); + insn.opcode = opcode;if (size >1 ) { insn.disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* -* Ensure that for JMP9 the displacement +* Ensure that for JMP8 the displacement * actually fits the signed byte. */ BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 ));
[tip: x86/core] x86/text-patching: Make text_gen_insn() play nice with ANNOTATE_NOENDBR
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: dbf45ea73312b99208809c3b7a4831d6bb4e6ed8 Gitweb: https://git.kernel.org/tip/dbf45ea73312b99208809c3b7a4831d6bb4e6ed8 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:19 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:28 +01:00
x86/text-patching: Make text_gen_insn() play nice with ANNOTATE_NOENDBR
--- arch/x86/include/asm/text-patching.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
arch/x86/include/asm/text-patching.h +9 -1
@@ -101,13 +101,21 @@ void *text_gen_insn(u8 opcode, const void *addr, const void *dest)static union text_poke_insn insn;/* per instance */ int size =text_opcode_size (opcode); +/* +* Hide the addresses to avoid the compiler folding in constants when +* referencing code, these can mess up annotations like +* ANNOTATE_NOENDBR. +*/ +OPTIMIZER_HIDE_VAR (addr); +OPTIMIZER_HIDE_VAR (dest); + insn.opcode = opcode;if (size >1 ) { insn.disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* -* Ensure that for JMP9 the displacement +* Ensure that for JMP8 the displacement * actually fits the signed byte. */ BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 ));
[tip: x86/core] x86/text-patching: Make text_gen_insn() play nice with ANNOTATE_NOENDBR
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: bbf92368b0b1fe472d489e62d3340d7897e9c697 Gitweb: https://git.kernel.org/tip/bbf92368b0b1fe472d489e62d3340d7897e9c697 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:19 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:33 +01:00
x86/text-patching: Make text_gen_insn() play nice with ANNOTATE_NOENDBR
--- arch/x86/include/asm/text-patching.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
arch/x86/include/asm/text-patching.h +9 -1
@@ -101,13 +101,21 @@ void *text_gen_insn(u8 opcode, const void *addr, const void *dest)static union text_poke_insn insn;/* per instance */ int size =text_opcode_size (opcode); +/* +* Hide the addresses to avoid the compiler folding in constants when +* referencing code, these can mess up annotations like +* ANNOTATE_NOENDBR. +*/ +OPTIMIZER_HIDE_VAR (addr); +OPTIMIZER_HIDE_VAR (dest); + insn.opcode = opcode;if (size >1 ) { insn.disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* -* Ensure that for JMP9 the displacement +* Ensure that for JMP8 the displacement * actually fits the signed byte. */ BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 ));
Peter Zijlstra
maintainerPATCH v4 9/45 x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch()
In reply to Peter ZijlstraLess duplication is more better.
--- arch/x86/include/asm/text-patching.h | 20 ++++++++++++++------ arch/x86/kernel/paravirt.c | 23 +++-------------------- 2 files changed, 17 insertions(+), 26 deletions(-)
arch/x86/include/asm/text-patching.h +14 -6
@@ -96,32 +96,40 @@ union text_poke_insn { };static __always_inline -void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +void __text_gen_insn(void *buf, u8 opcode, const void *addr, const void *dest, int size) { -static union text_poke_insn insn;/* per instance */ -int size =text_opcode_size (opcode); +union text_poke_insn *insn = buf; + +BUG_ON (size <text_opcode_size (opcode));/* * Hide the addresses to avoid the compiler folding in constants when * referencing code, these can mess up annotations like * ANNOTATE_NOENDBR. */ +OPTIMIZER_HIDE_VAR (insn);OPTIMIZER_HIDE_VAR (addr);OPTIMIZER_HIDE_VAR (dest); - insn.opcode = opcode; + insn->opcode = opcode;if (size >1 ) { - insn.disp = (long )dest - (long )(addr + size); + insn->disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* * Ensure that for JMP8 the displacement * actually fits the signed byte. */ -BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 )); +BUG_ON ((insn->disp >>31 ) != (insn->disp >>7 )); } } +} +static __always_inline +void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +{ +static union text_poke_insn insn;/* per instance */ +__text_gen_insn (&insn, opcode, addr, dest,text_opcode_size (opcode));return &insn.text; }
arch/x86/kernel/paravirt.c +3 -20
@@ -69,29 +69,12 @@ noinstr void paravirt_BUG(void)BUG (); } -struct branch { -unsigned char opcode; -u32 delta; -} __attribute__((packed)); -static unsigned paravirt_patch_call (void *insn_buff,const void *target,unsigned long addr,unsigned len) { -const int call_len =5 ; -struct branch *b = insn_buff; -unsigned long delta = (unsigned long )target - (addr+call_len); - -if (len < call_len) { -pr_warn ("paravirt: Failed to patch indirect CALL at %ps\n" , (void *)addr); -/* Kernel might not be viable if patching fails, bail out: */ -BUG_ON (1 ); - } - - b->opcode =0xe8 ;/* call */ - b->delta = delta; -BUILD_BUG_ON (sizeof (*b) != call_len); - -return call_len; +__text_gen_insn (insn_buff,CALL_INSN_OPCODE , + (void *)addr, target,CALL_INSN_SIZE ); +return CALL_INSN_SIZE ; }#ifdef CONFIG_PARAVIRT_XXL
[tip: x86/core] x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 7fb70b6975509102d848f1abdba394e0bdc122c1 Gitweb: https://git.kernel.org/tip/7fb70b6975509102d848f1abdba394e0bdc122c1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:20 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:28 +01:00
x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch()
Less duplication is more better.
--- arch/x86/include/asm/text-patching.h | 20 ++++++++++++++------ arch/x86/kernel/paravirt.c | 23 +++-------------------- 2 files changed, 17 insertions(+), 26 deletions(-)
arch/x86/include/asm/text-patching.h +14 -6
@@ -96,32 +96,40 @@ union text_poke_insn { };static __always_inline -void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +void __text_gen_insn(void *buf, u8 opcode, const void *addr, const void *dest, int size) { -static union text_poke_insn insn;/* per instance */ -int size =text_opcode_size (opcode); +union text_poke_insn *insn = buf; + +BUG_ON (size <text_opcode_size (opcode));/* * Hide the addresses to avoid the compiler folding in constants when * referencing code, these can mess up annotations like * ANNOTATE_NOENDBR. */ +OPTIMIZER_HIDE_VAR (insn);OPTIMIZER_HIDE_VAR (addr);OPTIMIZER_HIDE_VAR (dest); - insn.opcode = opcode; + insn->opcode = opcode;if (size >1 ) { - insn.disp = (long )dest - (long )(addr + size); + insn->disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* * Ensure that for JMP8 the displacement * actually fits the signed byte. */ -BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 )); +BUG_ON ((insn->disp >>31 ) != (insn->disp >>7 )); } } +} +static __always_inline +void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +{ +static union text_poke_insn insn;/* per instance */ +__text_gen_insn (&insn, opcode, addr, dest,text_opcode_size (opcode));return &insn.text; }
arch/x86/kernel/paravirt.c +3 -20
@@ -69,29 +69,12 @@ noinstr void paravirt_BUG(void)BUG (); } -struct branch { -unsigned char opcode; -u32 delta; -} __attribute__((packed)); -static unsigned paravirt_patch_call (void *insn_buff,const void *target,unsigned long addr,unsigned len) { -const int call_len =5 ; -struct branch *b = insn_buff; -unsigned long delta = (unsigned long )target - (addr+call_len); - -if (len < call_len) { -pr_warn ("paravirt: Failed to patch indirect CALL at %ps\n" , (void *)addr); -/* Kernel might not be viable if patching fails, bail out: */ -BUG_ON (1 ); - } - - b->opcode =0xe8 ;/* call */ - b->delta = delta; -BUILD_BUG_ON (sizeof (*b) != call_len); - -return call_len; +__text_gen_insn (insn_buff,CALL_INSN_OPCODE , + (void *)addr, target,CALL_INSN_SIZE ); +return CALL_INSN_SIZE ; }#ifdef CONFIG_PARAVIRT_XXL
[tip: x86/core] x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch()
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: ba27d1a80871eb8dbeddf34ec7d396c149cbb8d7 Gitweb: https://git.kernel.org/tip/ba27d1a80871eb8dbeddf34ec7d396c149cbb8d7 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:20 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:34 +01:00
x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch()
Less duplication is more better.
--- arch/x86/include/asm/text-patching.h | 20 ++++++++++++++------ arch/x86/kernel/paravirt.c | 23 +++-------------------- 2 files changed, 17 insertions(+), 26 deletions(-)
arch/x86/include/asm/text-patching.h +14 -6
@@ -96,32 +96,40 @@ union text_poke_insn { };static __always_inline -void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +void __text_gen_insn(void *buf, u8 opcode, const void *addr, const void *dest, int size) { -static union text_poke_insn insn;/* per instance */ -int size =text_opcode_size (opcode); +union text_poke_insn *insn = buf; + +BUG_ON (size <text_opcode_size (opcode));/* * Hide the addresses to avoid the compiler folding in constants when * referencing code, these can mess up annotations like * ANNOTATE_NOENDBR. */ +OPTIMIZER_HIDE_VAR (insn);OPTIMIZER_HIDE_VAR (addr);OPTIMIZER_HIDE_VAR (dest); - insn.opcode = opcode; + insn->opcode = opcode;if (size >1 ) { - insn.disp = (long )dest - (long )(addr + size); + insn->disp = (long )dest - (long )(addr + size);if (size ==2 ) {/* * Ensure that for JMP8 the displacement * actually fits the signed byte. */ -BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 )); +BUG_ON ((insn->disp >>31 ) != (insn->disp >>7 )); } } +} +static __always_inline +void *text_gen_insn(u8 opcode, const void *addr, const void *dest) +{ +static union text_poke_insn insn;/* per instance */ +__text_gen_insn (&insn, opcode, addr, dest,text_opcode_size (opcode));return &insn.text; }
arch/x86/kernel/paravirt.c +3 -20
@@ -69,29 +69,12 @@ noinstr void paravirt_BUG(void)BUG (); } -struct branch { -unsigned char opcode; -u32 delta; -} __attribute__((packed)); -static unsigned paravirt_patch_call (void *insn_buff,const void *target,unsigned long addr,unsigned len) { -const int call_len =5 ; -struct branch *b = insn_buff; -unsigned long delta = (unsigned long )target - (addr+call_len); - -if (len < call_len) { -pr_warn ("paravirt: Failed to patch indirect CALL at %ps\n" , (void *)addr); -/* Kernel might not be viable if patching fails, bail out: */ -BUG_ON (1 ); - } - - b->opcode =0xe8 ;/* call */ - b->delta = delta; -BUILD_BUG_ON (sizeof (*b) != call_len); - -return call_len; +__text_gen_insn (insn_buff,CALL_INSN_OPCODE , + (void *)addr, target,CALL_INSN_SIZE ); +return CALL_INSN_SIZE ; }#ifdef CONFIG_PARAVIRT_XXL
Peter Zijlstra
maintainerPATCH v4 10/45 x86/entry: Cleanup PARAVIRT
In reply to Peter ZijlstraSince commit 5c8f6a2e316e ("x86/xen: Add xenpv_restore_regs_and_return_to_usermode()") Xen will no longer reach this code and we can do away with the paravirt SWAPGS/INTERRUPT_RETURN.
--- arch/x86/entry/entry_64.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +2 -2
@@ -619,8 +619,8 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_ /* Restore RDI. */ popq %rdi - SWAPGS - INTERRUPT_RETURN + swapgs + jmp native_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
[tip: x86/core] x86/entry: Cleanup PARAVIRT
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 41c8dc098c627da1cba3b3dc471cba506414a7dd Gitweb: https://git.kernel.org/tip/41c8dc098c627da1cba3b3dc471cba506414a7dd Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:21 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:29 +01:00
x86/entry: Cleanup PARAVIRT
Since commit 5c8f6a2e316e ("x86/xen: Add xenpv_restore_regs_and_return_to_usermode()") Xen will no longer reach this code and we can do away with the paravirt SWAPGS/INTERRUPT_RETURN.
--- arch/x86/entry/entry_64.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +2 -2
@@ -608,8 +608,8 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi - SWAPGS - INTERRUPT_RETURN + swapgs + jmp native_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
[tip: x86/core] x86/entry: Cleanup PARAVIRT
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 6cf3e4c0d29102c74aca1ce0c1710be9d02e440e Gitweb: https://git.kernel.org/tip/6cf3e4c0d29102c74aca1ce0c1710be9d02e440e Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:21 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:34 +01:00
x86/entry: Cleanup PARAVIRT
Since commit 5c8f6a2e316e ("x86/xen: Add xenpv_restore_regs_and_return_to_usermode()") Xen will no longer reach this code and we can do away with the paravirt SWAPGS/INTERRUPT_RETURN.
--- arch/x86/entry/entry_64.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +2 -2
@@ -608,8 +608,8 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi - SWAPGS - INTERRUPT_RETURN + swapgs + jmp native_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
Peter Zijlstra
maintainerPATCH v4 11/45 x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
In reply to Peter ZijlstraBy doing an early rewrite of 'jmp native_iret` in restore_regs_and_return_to_kernel() we can get rid of the last INTERRUPT_RETURN user and paravirt_iret.
--- arch/x86/entry/entry_64.S | 11 ++++++++--- arch/x86/include/asm/irqflags.h | 5 ----- arch/x86/include/asm/paravirt_types.h | 1 - arch/x86/kernel/head_64.S | 3 ++- arch/x86/kernel/paravirt.c | 4 ---- arch/x86/xen/enlighten_pv.c | 7 ++++++- arch/x86/xen/xen-asm.S | 1 + 7 files changed, 17 insertions(+), 15 deletions(-)
arch/x86/entry/entry_64.S +8 -3
@@ -609,7 +609,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_ /* Restore RDI. */ popq %rdi swapgs - jmp native_iret + jmp .Lnative_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -626,9 +626,14 @@ SYM_INNER_LABEL(restore_regs_and_return_ * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization * when returning from IPI handler. */ - INTERRUPT_RETURN +#ifdef CONFIG_XEN_PV +SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL) + ANNOTATE_NOENDBR + .byte 0xe9 + .long .Lnative_iret - (. + 4) +#endif -SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) +.Lnative_iret: UNWIND_HINT_IRET_REGS /* * Are we returning to a stack segment from the LDT? Note: in
arch/x86/include/asm/irqflags.h +0 -5
@@ -141,13 +141,8 @@ static __always_inline void arch_local_i#ifdef CONFIG_X86_64 #ifdef CONFIG_XEN_PV #define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV -#define INTERRUPT_RETURN \ - ANNOTATE_RETPOLINE_SAFE; \ - ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ - X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")#else #define SWAPGS swapgs -#define INTERRUPT_RETURN jmp native_iret#endif #endif #endif /* !__ASSEMBLY__ */
arch/x86/include/asm/paravirt_types.h +0 -1
@@ -272,7 +272,6 @@ struct paravirt_patch_template {extern struct pv_info pv_info;extern struct paravirt_patch_template pv_ops; -extern void (*paravirt_iret)(void );#define PARAVIRT_PATCH (x) \ (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
arch/x86/kernel/head_64.S +2 -1
@@ -345,7 +345,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) /* Remove Error Code */ addq $8, %rsp - /* Pure iret required here - don't use INTERRUPT_RETURN */ iretq SYM_CODE_END(vc_boot_ghcb) #endif
@@ -426,6 +425,8 @@ SYM_CODE_END(early_idt_handler_common) * early_idt_handler_array can't be used because it returns via the * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. * + * XXX it does, fix this. + * * This handler will end up in the .init.text section and not be * available to boot secondary CPUs. */
arch/x86/kernel/paravirt.c +0 -4
@@ -132,8 +132,6 @@ void paravirt_set_sched_clock(u64 (*func }/* These are in entry.S */ -extern void native_iret (void ); -static struct resource reserve_ioports = { .start =0 , .end =IO_SPACE_LIMIT ,
@@ -397,8 +395,6 @@ struct paravirt_patch_template pv_ops =#ifdef CONFIG_PARAVIRT_XXL NOKPROBE_SYMBOL (native_load_idt); - -void (*paravirt_iret)(void ) = native_iret;#endif EXPORT_SYMBOL (pv_ops);
arch/x86/xen/enlighten_pv.c +6 -1
@@ -1177,6 +1177,8 @@ static void __init xen_domu_set_legacy_f x86_platform.legacy.rtc =0 ; } +extern void early_xen_iret_patch (void ); +/* First C function to be called on Xen boot */ asmlinkage __visible void __init xen_start_kernel(void) {
@@ -1187,6 +1189,10 @@ asmlinkage __visible void __init xen_staif (!xen_start_info)return ; +__text_gen_insn (&early_xen_iret_patch, +JMP32_INSN_OPCODE , &early_xen_iret_patch, &xen_iret, +JMP32_INSN_SIZE ); + xen_domain_type =XEN_PV_DOMAIN ; xen_start_flags = xen_start_info->flags;
@@ -1195,7 +1201,6 @@ asmlinkage __visible void __init xen_sta/* Install Xen paravirt ops */ pv_info = xen_info; pv_ops.cpu = xen_cpu_ops.cpu; - paravirt_iret = xen_iret;xen_init_irq_ops ();/*
arch/x86/xen/xen-asm.S +1 -0
@@ -189,6 +189,7 @@ hypercall_iret = hypercall_page + __HYPE */ SYM_CODE_START(xen_iret) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret SYM_CODE_END(xen_iret)
[tip: x86/core] x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 5b96185120bf588023760ac5bc9a0c96faf671e1 Gitweb: https://git.kernel.org/tip/5b96185120bf588023760ac5bc9a0c96faf671e1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:22 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:29 +01:00
x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
By doing an early rewrite of 'jmp native_iret` in restore_regs_and_return_to_kernel() we can get rid of the last INTERRUPT_RETURN user and paravirt_iret.
--- arch/x86/entry/entry_64.S | 11 ++++++++--- arch/x86/include/asm/irqflags.h | 5 ----- arch/x86/include/asm/paravirt_types.h | 1 - arch/x86/kernel/head_64.S | 3 ++- arch/x86/kernel/paravirt.c | 4 ---- arch/x86/xen/enlighten_pv.c | 7 ++++++- arch/x86/xen/xen-asm.S | 1 + 7 files changed, 17 insertions(+), 15 deletions(-)
arch/x86/entry/entry_64.S +8 -3
@@ -609,7 +609,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi swapgs - jmp native_iret + jmp .Lnative_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -626,9 +626,14 @@ SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization * when returning from IPI handler. */ - INTERRUPT_RETURN +#ifdef CONFIG_XEN_PV +SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL) + ANNOTATE_NOENDBR + .byte 0xe9 + .long .Lnative_iret - (. + 4) +#endif -SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) +.Lnative_iret: UNWIND_HINT_IRET_REGS /* * Are we returning to a stack segment from the LDT? Note: in
arch/x86/include/asm/irqflags.h +0 -5
@@ -141,13 +141,8 @@ static __always_inline void arch_local_irq_restore(unsigned long flags)#ifdef CONFIG_X86_64 #ifdef CONFIG_XEN_PV #define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV -#define INTERRUPT_RETURN \ - ANNOTATE_RETPOLINE_SAFE; \ - ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ - X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")#else #define SWAPGS swapgs -#define INTERRUPT_RETURN jmp native_iret#endif #endif #endif /* !__ASSEMBLY__ */
arch/x86/include/asm/paravirt_types.h +0 -1
@@ -272,7 +272,6 @@ struct paravirt_patch_template {extern struct pv_info pv_info;extern struct paravirt_patch_template pv_ops; -extern void (*paravirt_iret)(void );#define PARAVIRT_PATCH (x) \ (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
arch/x86/kernel/head_64.S +2 -1
@@ -345,7 +345,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) /* Remove Error Code */ addq $8, %rsp - /* Pure iret required here - don't use INTERRUPT_RETURN */ iretq SYM_CODE_END(vc_boot_ghcb) #endif
@@ -426,6 +425,8 @@ SYM_CODE_END(early_idt_handler_common) * early_idt_handler_array can't be used because it returns via the * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. * + * XXX it does, fix this. + * * This handler will end up in the .init.text section and not be * available to boot secondary CPUs. */
arch/x86/kernel/paravirt.c +0 -4
@@ -132,8 +132,6 @@ void paravirt_set_sched_clock(u64 (*func)(void)) }/* These are in entry.S */ -extern void native_iret (void ); -static struct resource reserve_ioports = { .start =0 , .end =IO_SPACE_LIMIT ,
@@ -397,8 +395,6 @@ struct paravirt_patch_template pv_ops = {#ifdef CONFIG_PARAVIRT_XXL NOKPROBE_SYMBOL (native_load_idt); - -void (*paravirt_iret)(void ) = native_iret;#endif EXPORT_SYMBOL (pv_ops);
arch/x86/xen/enlighten_pv.c +6 -1
@@ -1177,6 +1177,8 @@ static void __init xen_domu_set_legacy_features(void) x86_platform.legacy.rtc =0 ; } +extern void early_xen_iret_patch (void ); +/* First C function to be called on Xen boot */ asmlinkage __visible void __init xen_start_kernel(void) {
@@ -1187,6 +1189,10 @@ asmlinkage __visible void __init xen_start_kernel(void)if (!xen_start_info)return ; +__text_gen_insn (&early_xen_iret_patch, +JMP32_INSN_OPCODE , &early_xen_iret_patch, &xen_iret, +JMP32_INSN_SIZE ); + xen_domain_type =XEN_PV_DOMAIN ; xen_start_flags = xen_start_info->flags;
@@ -1195,7 +1201,6 @@ asmlinkage __visible void __init xen_start_kernel(void)/* Install Xen paravirt ops */ pv_info = xen_info; pv_ops.cpu = xen_cpu_ops.cpu; - paravirt_iret = xen_iret;xen_init_irq_ops ();/*
arch/x86/xen/xen-asm.S +1 -0
@@ -189,6 +189,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 */ SYM_CODE_START(xen_iret) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret SYM_CODE_END(xen_iret)
[tip: x86/core] x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 8b87d8cec1b31ea710568ae49ba5f5146318da0d Gitweb: https://git.kernel.org/tip/8b87d8cec1b31ea710568ae49ba5f5146318da0d Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:22 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:34 +01:00
x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
By doing an early rewrite of 'jmp native_iret` in restore_regs_and_return_to_kernel() we can get rid of the last INTERRUPT_RETURN user and paravirt_iret.
--- arch/x86/entry/entry_64.S | 11 ++++++++--- arch/x86/include/asm/irqflags.h | 5 ----- arch/x86/include/asm/paravirt_types.h | 1 - arch/x86/kernel/head_64.S | 3 ++- arch/x86/kernel/paravirt.c | 4 ---- arch/x86/xen/enlighten_pv.c | 7 ++++++- arch/x86/xen/xen-asm.S | 1 + 7 files changed, 17 insertions(+), 15 deletions(-)
arch/x86/entry/entry_64.S +8 -3
@@ -609,7 +609,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi swapgs - jmp native_iret + jmp .Lnative_iret SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -626,9 +626,14 @@ SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization * when returning from IPI handler. */ - INTERRUPT_RETURN +#ifdef CONFIG_XEN_PV +SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL) + ANNOTATE_NOENDBR + .byte 0xe9 + .long .Lnative_iret - (. + 4) +#endif -SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) +.Lnative_iret: UNWIND_HINT_IRET_REGS /* * Are we returning to a stack segment from the LDT? Note: in
arch/x86/include/asm/irqflags.h +0 -5
@@ -141,13 +141,8 @@ static __always_inline void arch_local_irq_restore(unsigned long flags)#ifdef CONFIG_X86_64 #ifdef CONFIG_XEN_PV #define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV -#define INTERRUPT_RETURN \ - ANNOTATE_RETPOLINE_SAFE; \ - ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ - X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")#else #define SWAPGS swapgs -#define INTERRUPT_RETURN jmp native_iret#endif #endif #endif /* !__ASSEMBLY__ */
arch/x86/include/asm/paravirt_types.h +0 -1
@@ -272,7 +272,6 @@ struct paravirt_patch_template {extern struct pv_info pv_info;extern struct paravirt_patch_template pv_ops; -extern void (*paravirt_iret)(void );#define PARAVIRT_PATCH (x) \ (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
arch/x86/kernel/head_64.S +2 -1
@@ -345,7 +345,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) /* Remove Error Code */ addq $8, %rsp - /* Pure iret required here - don't use INTERRUPT_RETURN */ iretq SYM_CODE_END(vc_boot_ghcb) #endif
@@ -426,6 +425,8 @@ SYM_CODE_END(early_idt_handler_common) * early_idt_handler_array can't be used because it returns via the * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early. * + * XXX it does, fix this. + * * This handler will end up in the .init.text section and not be * available to boot secondary CPUs. */
arch/x86/kernel/paravirt.c +0 -4
@@ -132,8 +132,6 @@ void paravirt_set_sched_clock(u64 (*func)(void)) }/* These are in entry.S */ -extern void native_iret (void ); -static struct resource reserve_ioports = { .start =0 , .end =IO_SPACE_LIMIT ,
@@ -397,8 +395,6 @@ struct paravirt_patch_template pv_ops = {#ifdef CONFIG_PARAVIRT_XXL NOKPROBE_SYMBOL (native_load_idt); - -void (*paravirt_iret)(void ) = native_iret;#endif EXPORT_SYMBOL (pv_ops);
arch/x86/xen/enlighten_pv.c +6 -1
@@ -1177,6 +1177,8 @@ static void __init xen_domu_set_legacy_features(void) x86_platform.legacy.rtc =0 ; } +extern void early_xen_iret_patch (void ); +/* First C function to be called on Xen boot */ asmlinkage __visible void __init xen_start_kernel(void) {
@@ -1187,6 +1189,10 @@ asmlinkage __visible void __init xen_start_kernel(void)if (!xen_start_info)return ; +__text_gen_insn (&early_xen_iret_patch, +JMP32_INSN_OPCODE , &early_xen_iret_patch, &xen_iret, +JMP32_INSN_SIZE ); + xen_domain_type =XEN_PV_DOMAIN ; xen_start_flags = xen_start_info->flags;
@@ -1195,7 +1201,6 @@ asmlinkage __visible void __init xen_start_kernel(void)/* Install Xen paravirt ops */ pv_info = xen_info; pv_ops.cpu = xen_cpu_ops.cpu; - paravirt_iret = xen_iret;xen_init_irq_ops ();/*
arch/x86/xen/xen-asm.S +1 -0
@@ -189,6 +189,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 */ SYM_CODE_START(xen_iret) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret SYM_CODE_END(xen_iret)
Peter Zijlstra
maintainerPATCH v4 12/45 x86/ibt,xen: Sprinkle the ENDBR
In reply to Peter ZijlstraEven though Xen currently doesn't advertise IBT, prepare for when it will eventually do so and sprinkle the ENDBR dust accordingly.
Even though most of the entry points are IRET like, the CPL0 Hypervisor can set WAIT-FOR-ENDBR and demand ENDBR at these sites.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/include/asm/segment.h | 2 +- arch/x86/kernel/head_64.S | 1 + arch/x86/xen/enlighten_pv.c | 3 +++ arch/x86/xen/xen-asm.S | 9 +++++++++ arch/x86/xen/xen-head.S | 8 ++++++-- 6 files changed, 21 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +1 -0
@@ -809,6 +809,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback */ SYM_CODE_START(xen_failsafe_callback) UNWIND_HINT_EMPTY + ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) jne 1f
arch/x86/include/asm/segment.h +1 -1
@@ -283,7 +283,7 @@ static inline void vdso_read_cpunode(uns* pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to * max 8 bytes. */ -#define XEN_EARLY_IDT_HANDLER_SIZE 8 +#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)#ifndef __ASSEMBLY__
arch/x86/kernel/head_64.S +1 -0
@@ -383,6 +383,7 @@ SYM_CODE_START(early_idt_handler_array) .endr UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) + ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) /*
arch/x86/xen/enlighten_pv.c +3 -0
@@ -624,6 +624,9 @@ static struct trap_array_entry trap_arraTRAP_ENTRY (exc_coprocessor_error, false ),TRAP_ENTRY (exc_alignment_check, false ),TRAP_ENTRY (exc_simd_coprocessor_error, false ), +#ifdef CONFIG_X86_KERNEL_IBT + TRAP_ENTRY(exc_control_protection, false ), +#endif };static bool __ref get_trap_addr (void **addr,unsigned int ist)
arch/x86/xen/xen-asm.S +9 -0
@@ -122,6 +122,7 @@ SYM_FUNC_END(xen_read_cr2_direct); .macro xen_pv_trap name SYM_CODE_START(xen_\name) UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp \name
@@ -147,6 +148,9 @@ xen_pv_trap asm_exc_page_fault xen_pv_trap asm_exc_spurious_interrupt_bug xen_pv_trap asm_exc_coprocessor_error xen_pv_trap asm_exc_alignment_check +#ifdef CONFIG_X86_KERNEL_IBT +xen_pv_trap asm_exc_control_protection +#endif #ifdef CONFIG_X86_MCE xen_pv_trap asm_xenpv_exc_machine_check #endif /* CONFIG_X86_MCE */
@@ -162,6 +166,7 @@ SYM_CODE_START(xen_early_idt_handler_arr i = 0 .rept NUM_EXCEPTION_VECTORS UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
@@ -231,6 +236,7 @@ SYM_CODE_END(xenpv_restore_regs_and_retu /* Normal 64-bit system call target */ SYM_CODE_START(xen_syscall_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -250,6 +256,7 @@ SYM_CODE_END(xen_syscall_target) /* 32-bit compat syscall target */ SYM_CODE_START(xen_syscall32_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -267,6 +274,7 @@ SYM_CODE_END(xen_syscall32_target) /* 32-bit compat sysenter target */ SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR /* * NB: Xen is polite and clears TF from EFLAGS for us. This means * that we don't need to guard against single step exceptions here.
@@ -290,6 +298,7 @@ SYM_CODE_END(xen_sysenter_target) SYM_CODE_START(xen_syscall32_target) SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $0
arch/x86/xen/xen-head.S +6 -2
@@ -25,8 +25,11 @@ SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) UNWIND_HINT_FUNC - .skip 31, 0x90 - RET + ANNOTATE_NOENDBR + /* + * Xen will write the hypercall page, and sort out ENDBR. + */ + .skip 32, 0xcc .endr #define HYPERCALL(n) \
@@ -74,6 +77,7 @@ SYM_CODE_END(startup_xen) .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) UNWIND_HINT_EMPTY + ENDBR call cpu_bringup_and_idle SYM_CODE_END(asm_cpu_bringup_and_idle)
[tip: x86/core] x86/ibt,xen: Sprinkle the ENDBR
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 745fc9ec4d0d007b72ae688525560ba39cdef784 Gitweb: https://git.kernel.org/tip/745fc9ec4d0d007b72ae688525560ba39cdef784 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:23 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:29 +01:00
x86/ibt,xen: Sprinkle the ENDBR
Even though Xen currently doesn't advertise IBT, prepare for when it will eventually do so and sprinkle the ENDBR dust accordingly.
Even though most of the entry points are IRET like, the CPL0 Hypervisor can set WAIT-FOR-ENDBR and demand ENDBR at these sites.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/include/asm/segment.h | 2 +- arch/x86/kernel/head_64.S | 1 + arch/x86/xen/enlighten_pv.c | 3 +++ arch/x86/xen/xen-asm.S | 9 +++++++++ arch/x86/xen/xen-head.S | 9 +++++++-- 6 files changed, 22 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +1 -0
@@ -809,6 +809,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) */ SYM_CODE_START(xen_failsafe_callback) UNWIND_HINT_EMPTY + ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) jne 1f
arch/x86/include/asm/segment.h +1 -1
@@ -283,7 +283,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)* pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to * max 8 bytes. */ -#define XEN_EARLY_IDT_HANDLER_SIZE 8 +#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)#ifndef __ASSEMBLY__
arch/x86/kernel/head_64.S +1 -0
@@ -383,6 +383,7 @@ SYM_CODE_START(early_idt_handler_array) .endr UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) + ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) /*
arch/x86/xen/enlighten_pv.c +3 -0
@@ -624,6 +624,9 @@ static struct trap_array_entry trap_array[] = {TRAP_ENTRY (exc_coprocessor_error, false ),TRAP_ENTRY (exc_alignment_check, false ),TRAP_ENTRY (exc_simd_coprocessor_error, false ), +#ifdef CONFIG_X86_KERNEL_IBT + TRAP_ENTRY(exc_control_protection, false ), +#endif };static bool __ref get_trap_addr (void **addr,unsigned int ist)
arch/x86/xen/xen-asm.S +9 -0
@@ -122,6 +122,7 @@ SYM_FUNC_END(xen_read_cr2_direct); .macro xen_pv_trap name SYM_CODE_START(xen_\name) UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp \name
@@ -147,6 +148,9 @@ xen_pv_trap asm_exc_page_fault xen_pv_trap asm_exc_spurious_interrupt_bug xen_pv_trap asm_exc_coprocessor_error xen_pv_trap asm_exc_alignment_check +#ifdef CONFIG_X86_KERNEL_IBT +xen_pv_trap asm_exc_control_protection +#endif #ifdef CONFIG_X86_MCE xen_pv_trap asm_xenpv_exc_machine_check #endif /* CONFIG_X86_MCE */
@@ -162,6 +166,7 @@ SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
@@ -231,6 +236,7 @@ SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode) /* Normal 64-bit system call target */ SYM_CODE_START(xen_syscall_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -250,6 +256,7 @@ SYM_CODE_END(xen_syscall_target) /* 32-bit compat syscall target */ SYM_CODE_START(xen_syscall32_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -267,6 +274,7 @@ SYM_CODE_END(xen_syscall32_target) /* 32-bit compat sysenter target */ SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR /* * NB: Xen is polite and clears TF from EFLAGS for us. This means * that we don't need to guard against single step exceptions here.
@@ -290,6 +298,7 @@ SYM_CODE_END(xen_sysenter_target) SYM_CODE_START(xen_syscall32_target) SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $0
arch/x86/xen/xen-head.S +7 -2
@@ -25,8 +25,12 @@ SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) UNWIND_HINT_FUNC - .skip 31, 0x90 - RET + ANNOTATE_NOENDBR + ret + /* + * Xen will write the hypercall page, and sort out ENDBR. + */ + .skip 31, 0xcc .endr #define HYPERCALL(n) \
@@ -74,6 +78,7 @@ SYM_CODE_END(startup_xen) .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) UNWIND_HINT_EMPTY + ENDBR call cpu_bringup_and_idle SYM_CODE_END(asm_cpu_bringup_and_idle)
[tip: x86/core] x86/ibt,xen: Sprinkle the ENDBR
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 5b2fc51576eff811a614e33cbbd0c3cb05022892 Gitweb: https://git.kernel.org/tip/5b2fc51576eff811a614e33cbbd0c3cb05022892 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:23 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:35 +01:00
x86/ibt,xen: Sprinkle the ENDBR
Even though Xen currently doesn't advertise IBT, prepare for when it will eventually do so and sprinkle the ENDBR dust accordingly.
Even though most of the entry points are IRET like, the CPL0 Hypervisor can set WAIT-FOR-ENDBR and demand ENDBR at these sites.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/include/asm/segment.h | 2 +- arch/x86/kernel/head_64.S | 1 + arch/x86/xen/enlighten_pv.c | 3 +++ arch/x86/xen/xen-asm.S | 9 +++++++++ arch/x86/xen/xen-head.S | 9 +++++++-- 6 files changed, 22 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +1 -0
@@ -809,6 +809,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) */ SYM_CODE_START(xen_failsafe_callback) UNWIND_HINT_EMPTY + ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) jne 1f
arch/x86/include/asm/segment.h +1 -1
@@ -283,7 +283,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)* pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to * max 8 bytes. */ -#define XEN_EARLY_IDT_HANDLER_SIZE 8 +#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)#ifndef __ASSEMBLY__
arch/x86/kernel/head_64.S +1 -0
@@ -383,6 +383,7 @@ SYM_CODE_START(early_idt_handler_array) .endr UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) + ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) /*
arch/x86/xen/enlighten_pv.c +3 -0
@@ -624,6 +624,9 @@ static struct trap_array_entry trap_array[] = {TRAP_ENTRY (exc_coprocessor_error, false ),TRAP_ENTRY (exc_alignment_check, false ),TRAP_ENTRY (exc_simd_coprocessor_error, false ), +#ifdef CONFIG_X86_KERNEL_IBT + TRAP_ENTRY(exc_control_protection, false ), +#endif };static bool __ref get_trap_addr (void **addr,unsigned int ist)
arch/x86/xen/xen-asm.S +9 -0
@@ -122,6 +122,7 @@ SYM_FUNC_END(xen_read_cr2_direct); .macro xen_pv_trap name SYM_CODE_START(xen_\name) UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp \name
@@ -147,6 +148,9 @@ xen_pv_trap asm_exc_page_fault xen_pv_trap asm_exc_spurious_interrupt_bug xen_pv_trap asm_exc_coprocessor_error xen_pv_trap asm_exc_alignment_check +#ifdef CONFIG_X86_KERNEL_IBT +xen_pv_trap asm_exc_control_protection +#endif #ifdef CONFIG_X86_MCE xen_pv_trap asm_xenpv_exc_machine_check #endif /* CONFIG_X86_MCE */
@@ -162,6 +166,7 @@ SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS UNWIND_HINT_EMPTY + ENDBR pop %rcx pop %r11 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
@@ -231,6 +236,7 @@ SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode) /* Normal 64-bit system call target */ SYM_CODE_START(xen_syscall_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -250,6 +256,7 @@ SYM_CODE_END(xen_syscall_target) /* 32-bit compat syscall target */ SYM_CODE_START(xen_syscall32_target) UNWIND_HINT_EMPTY + ENDBR popq %rcx popq %r11
@@ -267,6 +274,7 @@ SYM_CODE_END(xen_syscall32_target) /* 32-bit compat sysenter target */ SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR /* * NB: Xen is polite and clears TF from EFLAGS for us. This means * that we don't need to guard against single step exceptions here.
@@ -290,6 +298,7 @@ SYM_CODE_END(xen_sysenter_target) SYM_CODE_START(xen_syscall32_target) SYM_CODE_START(xen_sysenter_target) UNWIND_HINT_EMPTY + ENDBR lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $0
arch/x86/xen/xen-head.S +7 -2
@@ -25,8 +25,12 @@ SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) UNWIND_HINT_FUNC - .skip 31, 0x90 - RET + ANNOTATE_NOENDBR + ret + /* + * Xen will write the hypercall page, and sort out ENDBR. + */ + .skip 31, 0xcc .endr #define HYPERCALL(n) \
@@ -74,6 +78,7 @@ SYM_CODE_END(startup_xen) .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) UNWIND_HINT_EMPTY + ENDBR call cpu_bringup_and_idle SYM_CODE_END(asm_cpu_bringup_and_idle)
Peter Zijlstra
maintainerPATCH v4 13/45 x86/ibt,entry: Sprinkle ENDBR dust
In reply to Peter ZijlstraKernel entry points should be having ENDBR on for IBT configs.
The SYSCALL entry points are found through taking their respective address in order to program them in the MSRs, while the exception entry points are found through UNWIND_HINT_IRET_REGS.
The rule is that any UNWIND_HINT_IRET_REGS at sym+0 should have an ENDBR, see the later objtool ibt validation patch.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 3 +++ arch/x86/include/asm/idtentry.h | 20 +++++++++++--------- arch/x86/include/asm/segment.h | 3 ++- arch/x86/kernel/head_64.S | 4 +++- arch/x86/kernel/idt.c | 5 +++-- 6 files changed, 28 insertions(+), 13 deletions(-)
arch/x86/entry/entry_64.S +6 -0
@@ -86,6 +86,7 @@ SYM_CODE_START(entry_SYSCALL_64) UNWIND_HINT_EMPTY + ENDBR swapgs /* tss.sp2 is scratch space. */
@@ -350,6 +351,7 @@ SYM_CODE_END(ret_from_fork) .macro idtentry vector asmsym cfunc has_error_code:req SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + ENDBR ASM_CLAC .if \has_error_code == 0
@@ -417,6 +419,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -472,6 +475,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC /*
@@ -533,6 +537,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=8 + ENDBR ASM_CLAC /* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -1069,6 +1074,7 @@ SYM_CODE_END(error_return) */ SYM_CODE_START(asm_exc_nmi) UNWIND_HINT_IRET_REGS + ENDBR /* * We allow breakpoints in NMIs. If a breakpoint occurs, then
arch/x86/entry/entry_64_compat.S +3 -0
@@ -48,6 +48,7 @@ */ SYM_CODE_START(entry_SYSENTER_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ SWAPGS
@@ -198,6 +199,7 @@ SYM_CODE_END(entry_SYSENTER_compat) */ SYM_CODE_START(entry_SYSCALL_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ swapgs
@@ -340,6 +342,7 @@ SYM_CODE_END(entry_SYSCALL_compat) */ SYM_CODE_START(entry_INT80_compat) UNWIND_HINT_EMPTY + ENDBR /* * Interrupts are off on entry. */
arch/x86/include/asm/idtentry.h +11 -9
@@ -5,6 +5,8 @@/* Interrupts/Exceptions */ #include <asm/trapnr.h> +#define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT)) +#ifndef __ASSEMBLY__#include <linux/entry-common.h> #include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_re/* * ASM code to emit the common vector entry stubs where each stub is -* packed into 8 bytes. +* packed into IDT_ALIGN bytes. * * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because * GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_re* point is to mask off the bits above bit 7 because the push is sign * extending. */ - .align 8 + .align IDT_ALIGNSYM_CODE_START (irq_entries_start ) vector=FIRST_EXTERNAL_VECTOR .rept NR_EXTERNAL_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_common_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (irq_entries_start) #ifdef CONFIG_X86_LOCAL_APIC - .align 8 + .align IDT_ALIGNSYM_CODE_START (spurious_entries_start ) vector=FIRST_SYSTEM_VECTOR .rept NR_SYSTEM_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_spurious_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (spurious_entries_start)
arch/x86/include/asm/segment.h +2 -1
@@ -4,6 +4,7 @@#include <linux/const.h> #include <asm/alternative.h> +#include <asm/ibt.h> /* * Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(uns* vector has no error code (two bytes), a 'push $vector_number' (two * bytes), and a jump to the common entry code (up to five bytes). */ -#define EARLY_IDT_HANDLER_SIZE 9 +#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)/* * xen_early_idt_handler_array is for Xen pv guests: for each entry in
arch/x86/kernel/head_64.S +3 -1
@@ -371,9 +371,11 @@ SYM_CODE_START(early_idt_handler_array) .rept NUM_EXCEPTION_VECTORS .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0 UNWIND_HINT_IRET_REGS + ENDBR pushq $0 # Dummy error code, to make stack frame uniform .else UNWIND_HINT_IRET_REGS offset=8 + ENDBR .endif pushq $i # 72(%rsp) Vector number jmp early_idt_handler_common
@@ -381,11 +383,11 @@ SYM_CODE_START(early_idt_handler_array) i = i + 1 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc .endr - UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) + UNWIND_HINT_IRET_REGS offset=16 /* * The stack is the hardware frame, an error code or zero, and the * vector number.
arch/x86/kernel/idt.c +3 -2
@@ -10,6 +10,7 @@#include <asm/proto.h> #include <asm/desc.h> #include <asm/hw_irq.h> +#include <asm/idtentry.h> #define DPL0 0x0#define DPL3 0x3
@@ -272,7 +273,7 @@ void __init idt_setup_apic_and_irq_gatesidt_setup_from_table (idt_table, apic_idts,ARRAY_SIZE (apic_idts), true);for_each_clear_bit_from (i, system_vectors,FIRST_SYSTEM_VECTOR ) { - entry = irq_entries_start +8 * (i -FIRST_EXTERNAL_VECTOR ); + entry = irq_entries_start +IDT_ALIGN * (i -FIRST_EXTERNAL_VECTOR );set_intr_gate (i, entry); }
@@ -283,7 +284,7 @@ void __init idt_setup_apic_and_irq_gates* system_vectors bitmap. Otherwise they show up in * /proc/interrupts. */ - entry = spurious_entries_start +8 * (i -FIRST_SYSTEM_VECTOR ); + entry = spurious_entries_start +IDT_ALIGN * (i -FIRST_SYSTEM_VECTOR ); set_intr_gate(i, entry); }#endif
[tip: x86/core] x86/ibt,entry: Sprinkle ENDBR dust
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 32850ea3d35102af3aa2cfbcb71744e9f54de1f8 Gitweb: https://git.kernel.org/tip/32850ea3d35102af3aa2cfbcb71744e9f54de1f8 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:24 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:30 +01:00
x86/ibt,entry: Sprinkle ENDBR dust
Kernel entry points should be having ENDBR on for IBT configs.
The SYSCALL entry points are found through taking their respective address in order to program them in the MSRs, while the exception entry points are found through UNWIND_HINT_IRET_REGS.
The rule is that any UNWIND_HINT_IRET_REGS at sym+0 should have an ENDBR, see the later objtool ibt validation patch.
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 3 +++ arch/x86/include/asm/idtentry.h | 20 +++++++++++--------- arch/x86/include/asm/segment.h | 3 ++- arch/x86/kernel/head_64.S | 4 +++- arch/x86/kernel/idt.c | 5 +++-- 6 files changed, 28 insertions(+), 13 deletions(-)
arch/x86/entry/entry_64.S +6 -0
@@ -86,6 +86,7 @@ SYM_CODE_START(entry_SYSCALL_64) UNWIND_HINT_EMPTY + ENDBR swapgs /* tss.sp2 is scratch space. */
@@ -350,6 +351,7 @@ SYM_CODE_END(ret_from_fork) .macro idtentry vector asmsym cfunc has_error_code:req SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + ENDBR ASM_CLAC .if \has_error_code == 0
@@ -417,6 +419,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -472,6 +475,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC /*
@@ -533,6 +537,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=8 + ENDBR ASM_CLAC /* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -1069,6 +1074,7 @@ SYM_CODE_END(error_return) */ SYM_CODE_START(asm_exc_nmi) UNWIND_HINT_IRET_REGS + ENDBR /* * We allow breakpoints in NMIs. If a breakpoint occurs, then
arch/x86/entry/entry_64_compat.S +3 -0
@@ -48,6 +48,7 @@ */ SYM_CODE_START(entry_SYSENTER_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ SWAPGS
@@ -198,6 +199,7 @@ SYM_CODE_END(entry_SYSENTER_compat) */ SYM_CODE_START(entry_SYSCALL_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ swapgs
@@ -340,6 +342,7 @@ SYM_CODE_END(entry_SYSCALL_compat) */ SYM_CODE_START(entry_INT80_compat) UNWIND_HINT_EMPTY + ENDBR /* * Interrupts are off on entry. */
arch/x86/include/asm/idtentry.h +11 -9
@@ -5,6 +5,8 @@/* Interrupts/Exceptions */ #include <asm/trapnr.h> +#define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT)) +#ifndef __ASSEMBLY__#include <linux/entry-common.h> #include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_regs *regs, \/* * ASM code to emit the common vector entry stubs where each stub is -* packed into 8 bytes. +* packed into IDT_ALIGN bytes. * * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because * GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_regs *regs, \* point is to mask off the bits above bit 7 because the push is sign * extending. */ - .align 8 + .align IDT_ALIGNSYM_CODE_START (irq_entries_start ) vector=FIRST_EXTERNAL_VECTOR .rept NR_EXTERNAL_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_common_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (irq_entries_start) #ifdef CONFIG_X86_LOCAL_APIC - .align 8 + .align IDT_ALIGNSYM_CODE_START (spurious_entries_start ) vector=FIRST_SYSTEM_VECTOR .rept NR_SYSTEM_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_spurious_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (spurious_entries_start)
arch/x86/include/asm/segment.h +2 -1
@@ -4,6 +4,7 @@#include <linux/const.h> #include <asm/alternative.h> +#include <asm/ibt.h> /* * Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)* vector has no error code (two bytes), a 'push $vector_number' (two * bytes), and a jump to the common entry code (up to five bytes). */ -#define EARLY_IDT_HANDLER_SIZE 9 +#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)/* * xen_early_idt_handler_array is for Xen pv guests: for each entry in
arch/x86/kernel/head_64.S +3 -1
@@ -371,9 +371,11 @@ SYM_CODE_START(early_idt_handler_array) .rept NUM_EXCEPTION_VECTORS .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0 UNWIND_HINT_IRET_REGS + ENDBR pushq $0 # Dummy error code, to make stack frame uniform .else UNWIND_HINT_IRET_REGS offset=8 + ENDBR .endif pushq $i # 72(%rsp) Vector number jmp early_idt_handler_common
@@ -381,11 +383,11 @@ SYM_CODE_START(early_idt_handler_array) i = i + 1 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc .endr - UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) + UNWIND_HINT_IRET_REGS offset=16 /* * The stack is the hardware frame, an error code or zero, and the * vector number.
arch/x86/kernel/idt.c +3 -2
@@ -10,6 +10,7 @@#include <asm/proto.h> #include <asm/desc.h> #include <asm/hw_irq.h> +#include <asm/idtentry.h> #define DPL0 0x0#define DPL3 0x3
@@ -272,7 +273,7 @@ void __init idt_setup_apic_and_irq_gates(void)idt_setup_from_table (idt_table, apic_idts,ARRAY_SIZE (apic_idts), true);for_each_clear_bit_from (i, system_vectors,FIRST_SYSTEM_VECTOR ) { - entry = irq_entries_start +8 * (i -FIRST_EXTERNAL_VECTOR ); + entry = irq_entries_start +IDT_ALIGN * (i -FIRST_EXTERNAL_VECTOR );set_intr_gate (i, entry); }
@@ -283,7 +284,7 @@ void __init idt_setup_apic_and_irq_gates(void)* system_vectors bitmap. Otherwise they show up in * /proc/interrupts. */ - entry = spurious_entries_start +8 * (i -FIRST_SYSTEM_VECTOR ); + entry = spurious_entries_start +IDT_ALIGN * (i -FIRST_SYSTEM_VECTOR ); set_intr_gate(i, entry); }#endif
[tip: x86/core] x86/ibt,entry: Sprinkle ENDBR dust
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 8f93402b92d443573d310250efa0b7f352fec992 Gitweb: https://git.kernel.org/tip/8f93402b92d443573d310250efa0b7f352fec992 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:24 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:35 +01:00
x86/ibt,entry: Sprinkle ENDBR dust
Kernel entry points should be having ENDBR on for IBT configs.
The SYSCALL entry points are found through taking their respective address in order to program them in the MSRs, while the exception entry points are found through UNWIND_HINT_IRET_REGS.
The rule is that any UNWIND_HINT_IRET_REGS at sym+0 should have an ENDBR, see the later objtool ibt validation patch.
--- arch/x86/entry/entry_64.S | 7 +++++++ arch/x86/entry/entry_64_compat.S | 3 +++ arch/x86/include/asm/idtentry.h | 20 +++++++++++--------- arch/x86/include/asm/segment.h | 3 ++- arch/x86/kernel/head_64.S | 4 +++- arch/x86/kernel/idt.c | 5 +++-- 6 files changed, 29 insertions(+), 13 deletions(-)
arch/x86/entry/entry_64.S +7 -0
@@ -86,6 +86,7 @@ SYM_CODE_START(entry_SYSCALL_64) UNWIND_HINT_EMPTY + ENDBR swapgs /* tss.sp2 is scratch space. */
@@ -350,6 +351,7 @@ SYM_CODE_END(ret_from_fork) .macro idtentry vector asmsym cfunc has_error_code:req SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + ENDBR ASM_CLAC .if \has_error_code == 0
@@ -417,6 +419,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -472,6 +475,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS + ENDBR ASM_CLAC /*
@@ -533,6 +537,7 @@ SYM_CODE_END(\asmsym) .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=8 + ENDBR ASM_CLAC /* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -1069,6 +1074,7 @@ SYM_CODE_END(error_return) */ SYM_CODE_START(asm_exc_nmi) UNWIND_HINT_IRET_REGS + ENDBR /* * We allow breakpoints in NMIs. If a breakpoint occurs, then
@@ -1427,6 +1433,7 @@ SYM_CODE_END(asm_exc_nmi) */ SYM_CODE_START(ignore_sysret) UNWIND_HINT_EMPTY + ENDBR mov $-ENOSYS, %eax sysretl SYM_CODE_END(ignore_sysret)
arch/x86/entry/entry_64_compat.S +3 -0
@@ -48,6 +48,7 @@ */ SYM_CODE_START(entry_SYSENTER_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ SWAPGS
@@ -198,6 +199,7 @@ SYM_CODE_END(entry_SYSENTER_compat) */ SYM_CODE_START(entry_SYSCALL_compat) UNWIND_HINT_EMPTY + ENDBR /* Interrupts are off on entry. */ swapgs
@@ -340,6 +342,7 @@ SYM_CODE_END(entry_SYSCALL_compat) */ SYM_CODE_START(entry_INT80_compat) UNWIND_HINT_EMPTY + ENDBR /* * Interrupts are off on entry. */
arch/x86/include/asm/idtentry.h +11 -9
@@ -5,6 +5,8 @@/* Interrupts/Exceptions */ #include <asm/trapnr.h> +#define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT)) +#ifndef __ASSEMBLY__#include <linux/entry-common.h> #include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_regs *regs, \/* * ASM code to emit the common vector entry stubs where each stub is -* packed into 8 bytes. +* packed into IDT_ALIGN bytes. * * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because * GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_regs *regs, \* point is to mask off the bits above bit 7 because the push is sign * extending. */ - .align 8 + .align IDT_ALIGNSYM_CODE_START (irq_entries_start ) vector=FIRST_EXTERNAL_VECTOR .rept NR_EXTERNAL_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_common_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (irq_entries_start) #ifdef CONFIG_X86_LOCAL_APIC - .align 8 + .align IDT_ALIGNSYM_CODE_START (spurious_entries_start ) vector=FIRST_SYSTEM_VECTOR .rept NR_SYSTEM_VECTORSUNWIND_HINT_IRET_REGS 0 : + ENDBR .byte 0x6a, vector jmp asm_spurious_interrupt - nop - /* Ensure that the above is 8 bytes max */ - . = 0b + 8 + /* Ensure that the above is IDT_ALIGN bytes max */ + .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 .endrSYM_CODE_END (spurious_entries_start)
arch/x86/include/asm/segment.h +2 -1
@@ -4,6 +4,7 @@#include <linux/const.h> #include <asm/alternative.h> +#include <asm/ibt.h> /* * Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)* vector has no error code (two bytes), a 'push $vector_number' (two * bytes), and a jump to the common entry code (up to five bytes). */ -#define EARLY_IDT_HANDLER_SIZE 9 +#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)/* * xen_early_idt_handler_array is for Xen pv guests: for each entry in
arch/x86/kernel/head_64.S +3 -1
@@ -371,9 +371,11 @@ SYM_CODE_START(early_idt_handler_array) .rept NUM_EXCEPTION_VECTORS .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0 UNWIND_HINT_IRET_REGS + ENDBR pushq $0 # Dummy error code, to make stack frame uniform .else UNWIND_HINT_IRET_REGS offset=8 + ENDBR .endif pushq $i # 72(%rsp) Vector number jmp early_idt_handler_common
@@ -381,11 +383,11 @@ SYM_CODE_START(early_idt_handler_array) i = i + 1 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc .endr - UNWIND_HINT_IRET_REGS offset=16 SYM_CODE_END(early_idt_handler_array) ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS] SYM_CODE_START_LOCAL(early_idt_handler_common) + UNWIND_HINT_IRET_REGS offset=16 /* * The stack is the hardware frame, an error code or zero, and the * vector number.
arch/x86/kernel/idt.c +3 -2
@@ -10,6 +10,7 @@#include <asm/proto.h> #include <asm/desc.h> #include <asm/hw_irq.h> +#include <asm/idtentry.h> #define DPL0 0x0#define DPL3 0x3
@@ -272,7 +273,7 @@ void __init idt_setup_apic_and_irq_gates(void)idt_setup_from_table (idt_table, apic_idts,ARRAY_SIZE (apic_idts), true);for_each_clear_bit_from (i, system_vectors,FIRST_SYSTEM_VECTOR ) { - entry = irq_entries_start +8 * (i -FIRST_EXTERNAL_VECTOR ); + entry = irq_entries_start +IDT_ALIGN * (i -FIRST_EXTERNAL_VECTOR );set_intr_gate (i, entry); }
@@ -283,7 +284,7 @@ void __init idt_setup_apic_and_irq_gates(void)* system_vectors bitmap. Otherwise they show up in * /proc/interrupts. */ - entry = spurious_entries_start +8 * (i -FIRST_SYSTEM_VECTOR ); + entry = spurious_entries_start +IDT_ALIGN * (i -FIRST_SYSTEM_VECTOR ); set_intr_gate(i, entry); }#endif
Peter Zijlstra
maintainerPATCH v4 14/45 x86/linkage: Add ENDBR to SYM_FUNC_START*()
In reply to Peter ZijlstraEnsure the ASM functions have ENDBR on for IBT builds, this follows the ARM64 example. Unlike ARM64, we'll likely end up overwriting them with poison.
--- arch/x86/include/asm/linkage.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
arch/x86/include/asm/linkage.h +31 -0
@@ -3,6 +3,7 @@#define _ASM_X86_LINKAGE_H#include <linux/stringify.h> +#include <asm/ibt.h> #undef notrace#define notrace __attribute__((no_instrument_function))
@@ -34,5 +35,35 @@#endif /* __ASSEMBLY__ */ +/* SYM_FUNC_START -- use for global functions */ +#define SYM_FUNC_START (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ +#define SYM_FUNC_START_NOALIGN (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_LOCAL -- use for local functions */ +#define SYM_FUNC_START_LOCAL (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ +#define SYM_FUNC_START_LOCAL_NOALIGN (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_WEAK -- use for weak functions */ +#define SYM_FUNC_START_WEAK (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ +#define SYM_FUNC_START_WEAK_NOALIGN (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ + ENDBR +#endif /* _ASM_X86_LINKAGE_H */
[tip: x86/core] x86/linkage: Add ENDBR to SYM_FUNC_START*()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 1d295d6e83f8ce357cafe11a86dab1deecb13963 Gitweb: https://git.kernel.org/tip/1d295d6e83f8ce357cafe11a86dab1deecb13963 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:25 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:30 +01:00
x86/linkage: Add ENDBR to SYM_FUNC_START*()
Ensure the ASM functions have ENDBR on for IBT builds, this follows the ARM64 example. Unlike ARM64, we'll likely end up overwriting them with poison.
--- arch/x86/include/asm/linkage.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
arch/x86/include/asm/linkage.h +31 -0
@@ -3,6 +3,7 @@#define _ASM_X86_LINKAGE_H#include <linux/stringify.h> +#include <asm/ibt.h> #undef notrace#define notrace __attribute__((no_instrument_function))
@@ -34,5 +35,35 @@#endif /* __ASSEMBLY__ */ +/* SYM_FUNC_START -- use for global functions */ +#define SYM_FUNC_START (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ +#define SYM_FUNC_START_NOALIGN (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_LOCAL -- use for local functions */ +#define SYM_FUNC_START_LOCAL (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ +#define SYM_FUNC_START_LOCAL_NOALIGN (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_WEAK -- use for weak functions */ +#define SYM_FUNC_START_WEAK (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ +#define SYM_FUNC_START_WEAK_NOALIGN (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ + ENDBR +#endif /* _ASM_X86_LINKAGE_H */
[tip: x86/core] x86/linkage: Add ENDBR to SYM_FUNC_START*()
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c4691712b546b93707a408368e3c57ae870a2dc4 Gitweb: https://git.kernel.org/tip/c4691712b546b93707a408368e3c57ae870a2dc4 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:25 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:36 +01:00
x86/linkage: Add ENDBR to SYM_FUNC_START*()
Ensure the ASM functions have ENDBR on for IBT builds, this follows the ARM64 example. Unlike ARM64, we'll likely end up overwriting them with poison.
--- arch/x86/include/asm/linkage.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
arch/x86/include/asm/linkage.h +31 -0
@@ -3,6 +3,7 @@#define _ASM_X86_LINKAGE_H#include <linux/stringify.h> +#include <asm/ibt.h> #undef notrace#define notrace __attribute__((no_instrument_function))
@@ -34,5 +35,35 @@#endif /* __ASSEMBLY__ */ +/* SYM_FUNC_START -- use for global functions */ +#define SYM_FUNC_START (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ +#define SYM_FUNC_START_NOALIGN (name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_LOCAL -- use for local functions */ +#define SYM_FUNC_START_LOCAL (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ +#define SYM_FUNC_START_LOCAL_NOALIGN (name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ + ENDBR + +/* SYM_FUNC_START_WEAK -- use for weak functions */ +#define SYM_FUNC_START_WEAK (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \ + ENDBR + +/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ +#define SYM_FUNC_START_WEAK_NOALIGN (name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ + ENDBR +#endif /* _ASM_X86_LINKAGE_H */
Peter Zijlstra
maintainerPATCH v4 15/45 x86/ibt,paravirt: Sprinkle ENDBR
In reply to Peter Zijlstra--- arch/x86/include/asm/paravirt.h | 1 + arch/x86/include/asm/qspinlock_paravirt.h | 3 +++ arch/x86/kernel/kvm.c | 3 ++- arch/x86/kernel/paravirt.c | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-)
arch/x86/include/asm/paravirt.h +1 -0
@@ -666,6 +666,7 @@ bool __raw_callee_save___native_vcpu_is_ ".globl " PV_THUNK_NAME(func) ";" \ ".type " PV_THUNK_NAME(func) ", @function;" \ PV_THUNK_NAME(func) ":" \ +ASM_ENDBR \FRAME_BEGIN \PV_SAVE_ALL_CALLER_REGS \ "call " #func ";" \
arch/x86/include/asm/qspinlock_paravirt.h +3 -0
@@ -2,6 +2,8 @@#ifndef __ASM_QSPINLOCK_PARAVIRT_H#define __ASM_QSPINLOCK_PARAVIRT_H +#include <asm/ibt.h> +/* * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit * registers. For i386, however, only 1 32-bit register needs to be saved
@@ -39,6 +41,7 @@ asm (".pushsection .text;"".type " PV_UNLOCK ", @function;" ".align 4,0x90;" PV_UNLOCK ": " +ASM_ENDBR FRAME_BEGIN "push %rdx;" "mov $0x1,%eax;"
arch/x86/kernel/kvm.c +2 -1
@@ -1024,10 +1024,11 @@ asm(".global __raw_callee_save___kvm_vcpu_is_preempted;" ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" "__raw_callee_save___kvm_vcpu_is_preempted:" +ASM_ENDBR "movq __per_cpu_offset(,%rdi,8), %rax;" "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);""setne %al;" -"ret;" +ASM_RET ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" ".popsection");
arch/x86/kernel/paravirt.c +2 -0
@@ -41,6 +41,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" +ASM_ENDBR ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t"
@@ -50,6 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n asm (".pushsection .entry.text, \"ax\"\n" ".global paravirt_ret0\n" "paravirt_ret0:\n\t" +ASM_ENDBR "xor %" _ASM_AX", %" _ASM_AX";\n\t" ASM_RET ".size paravirt_ret0, . - paravirt_ret0\n\t"
[tip: x86/core] x86/ibt,paravirt: Sprinkle ENDBR
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f0a38bd300b266aa78d8384995a1d81f6d7a95ff Gitweb: https://git.kernel.org/tip/f0a38bd300b266aa78d8384995a1d81f6d7a95ff Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:26 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:30 +01:00
x86/ibt,paravirt: Sprinkle ENDBR
--- arch/x86/include/asm/paravirt.h | 1 + arch/x86/include/asm/qspinlock_paravirt.h | 3 +++ arch/x86/kernel/kvm.c | 3 ++- arch/x86/kernel/paravirt.c | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-)
arch/x86/include/asm/paravirt.h +1 -0
@@ -666,6 +666,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); ".globl " PV_THUNK_NAME(func) ";" \ ".type " PV_THUNK_NAME(func) ", @function;" \ PV_THUNK_NAME(func) ":" \ +ASM_ENDBR \FRAME_BEGIN \PV_SAVE_ALL_CALLER_REGS \ "call " #func ";" \
arch/x86/include/asm/qspinlock_paravirt.h +3 -0
@@ -2,6 +2,8 @@#ifndef __ASM_QSPINLOCK_PARAVIRT_H#define __ASM_QSPINLOCK_PARAVIRT_H +#include <asm/ibt.h> +/* * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit * registers. For i386, however, only 1 32-bit register needs to be saved
@@ -39,6 +41,7 @@ asm (".pushsection .text;"".type " PV_UNLOCK ", @function;" ".align 4,0x90;" PV_UNLOCK ": " +ASM_ENDBR FRAME_BEGIN "push %rdx;" "mov $0x1,%eax;"
arch/x86/kernel/kvm.c +2 -1
@@ -1029,10 +1029,11 @@ asm(".global __raw_callee_save___kvm_vcpu_is_preempted;" ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" "__raw_callee_save___kvm_vcpu_is_preempted:" +ASM_ENDBR "movq __per_cpu_offset(,%rdi,8), %rax;" "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);""setne %al;" -"ret;" +ASM_RET ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" ".popsection");
arch/x86/kernel/paravirt.c +2 -0
@@ -41,6 +41,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" +ASM_ENDBR ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t"
@@ -50,6 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n" asm (".pushsection .entry.text, \"ax\"\n" ".global paravirt_ret0\n" "paravirt_ret0:\n\t" +ASM_ENDBR "xor %" _ASM_AX", %" _ASM_AX";\n\t" ASM_RET ".size paravirt_ret0, . - paravirt_ret0\n\t"
[tip: x86/core] x86/ibt,paravirt: Sprinkle ENDBR
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c3b037917c6a4cbb09ab2d6ccf19f02335ad1847 Gitweb: https://git.kernel.org/tip/c3b037917c6a4cbb09ab2d6ccf19f02335ad1847 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:26 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:36 +01:00
x86/ibt,paravirt: Sprinkle ENDBR
--- arch/x86/include/asm/paravirt.h | 1 + arch/x86/include/asm/qspinlock_paravirt.h | 3 +++ arch/x86/kernel/kvm.c | 3 ++- arch/x86/kernel/paravirt.c | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-)
arch/x86/include/asm/paravirt.h +1 -0
@@ -666,6 +666,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); ".globl " PV_THUNK_NAME(func) ";" \ ".type " PV_THUNK_NAME(func) ", @function;" \ PV_THUNK_NAME(func) ":" \ +ASM_ENDBR \FRAME_BEGIN \PV_SAVE_ALL_CALLER_REGS \ "call " #func ";" \
arch/x86/include/asm/qspinlock_paravirt.h +3 -0
@@ -2,6 +2,8 @@#ifndef __ASM_QSPINLOCK_PARAVIRT_H#define __ASM_QSPINLOCK_PARAVIRT_H +#include <asm/ibt.h> +/* * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit * registers. For i386, however, only 1 32-bit register needs to be saved
@@ -39,6 +41,7 @@ asm (".pushsection .text;"".type " PV_UNLOCK ", @function;" ".align 4,0x90;" PV_UNLOCK ": " +ASM_ENDBR FRAME_BEGIN "push %rdx;" "mov $0x1,%eax;"
arch/x86/kernel/kvm.c +2 -1
@@ -1029,10 +1029,11 @@ asm(".global __raw_callee_save___kvm_vcpu_is_preempted;" ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" "__raw_callee_save___kvm_vcpu_is_preempted:" +ASM_ENDBR "movq __per_cpu_offset(,%rdi,8), %rax;" "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);""setne %al;" -"ret;" +ASM_RET ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" ".popsection");
arch/x86/kernel/paravirt.c +2 -0
@@ -41,6 +41,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" +ASM_ENDBR ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t"
@@ -50,6 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n" asm (".pushsection .entry.text, \"ax\"\n" ".global paravirt_ret0\n" "paravirt_ret0:\n\t" +ASM_ENDBR "xor %" _ASM_AX", %" _ASM_AX";\n\t" ASM_RET ".size paravirt_ret0, . - paravirt_ret0\n\t"
Peter Zijlstra
maintainerPATCH v4 16/45 x86/ibt,crypto: Add ENDBR for the jump-table entries
In reply to Peter ZijlstraThe code does:
## branch into array mov jump_table(,%rax,8), %bufp JMP_NOSPEC bufp
resulting in needing to mark the jump-table entries with ENDBR.
--- arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 3 +++ 1 file changed, 3 insertions(+)
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +3 -0
@@ -195,6 +195,7 @@ SYM_FUNC_START(crc_pcl) .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 crc32q -i*8(block_2), crc2
@@ -204,6 +205,7 @@ LABEL crc_ %i .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 # SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet
@@ -237,6 +239,7 @@ LABEL crc_ %i ################################################################ LABEL crc_ 0 + ENDBR mov tmp, len cmp $128*24, tmp jae full_block
[tip: x86/core] x86/ibt,crypto: Add ENDBR for the jump-table entries
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: db76667e784428a927cc86b5a6d0cbade416c9e8 Gitweb: https://git.kernel.org/tip/db76667e784428a927cc86b5a6d0cbade416c9e8 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:27 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:31 +01:00
x86/ibt,crypto: Add ENDBR for the jump-table entries
The code does:
## branch into array mov jump_table(,%rax,8), %bufp JMP_NOSPEC bufp
resulting in needing to mark the jump-table entries with ENDBR.
--- arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 3 +++ 1 file changed, 3 insertions(+)
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +3 -0
@@ -195,6 +195,7 @@ crc_array: .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 crc32q -i*8(block_2), crc2
@@ -204,6 +205,7 @@ LABEL crc_ %i .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 # SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet
@@ -237,6 +239,7 @@ LABEL crc_ %i ################################################################ LABEL crc_ 0 + ENDBR mov tmp, len cmp $128*24, tmp jae full_block
[tip: x86/core] x86/ibt,crypto: Add ENDBR for the jump-table entries
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 214b9a83b617367d53680812ad09687542370b8e Gitweb: https://git.kernel.org/tip/214b9a83b617367d53680812ad09687542370b8e Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:27 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:36 +01:00
x86/ibt,crypto: Add ENDBR for the jump-table entries
The code does:
## branch into array mov jump_table(,%rax,8), %bufp JMP_NOSPEC bufp
resulting in needing to mark the jump-table entries with ENDBR.
--- arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 3 +++ 1 file changed, 3 insertions(+)
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +3 -0
@@ -195,6 +195,7 @@ crc_array: .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 crc32q -i*8(block_2), crc2
@@ -204,6 +205,7 @@ LABEL crc_ %i .altmacro LABEL crc_ %i .noaltmacro + ENDBR crc32q -i*8(block_0), crc_init crc32q -i*8(block_1), crc1 # SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet
@@ -237,6 +239,7 @@ LABEL crc_ %i ################################################################ LABEL crc_ 0 + ENDBR mov tmp, len cmp $128*24, tmp jae full_block
Peter Zijlstra
maintainerPATCH v4 17/45 x86/ibt,kvm: Add ENDBR to fastops
In reply to Peter Zijlstra--- arch/x86/kvm/emulate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
arch/x86/kvm/emulate.c +4 -2
@@ -189,7 +189,7 @@ #define X16(x...) X8(x), X8(x)#define NR_FASTOP (ilog2(sizeof(ulong)) + 1) -#define FASTOP_SIZE 8 +#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))struct opcode {u64 flags;
@@ -311,7 +311,8 @@ static int fastop(struct x86_emulate_ctx#define __FOP_FUNC (name) \ ".align " __stringify(FASTOP_SIZE) " \n\t" \ ".type " name ", @function \n\t" \ - name ":\n\t" + name ":\n\t" \ + ASM_ENDBR#define FOP_FUNC (name) \ __FOP_FUNC(#name)
@@ -433,6 +434,7 @@ static int fastop(struct x86_emulate_ctx".align 4 \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ + ASM_ENDBR \ #op " %al \n\t" \ __FOP_RET(#op)
[tip: x86/core] x86/ibt,kvm: Add ENDBR to fastops
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 598b53d261c1f9aeedc375a66e5e77ef952bb857 Gitweb: https://git.kernel.org/tip/598b53d261c1f9aeedc375a66e5e77ef952bb857 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:28 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:31 +01:00
x86/ibt,kvm: Add ENDBR to fastops
--- arch/x86/kvm/emulate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
arch/x86/kvm/emulate.c +4 -2
@@ -189,7 +189,7 @@ #define X16(x...) X8(x), X8(x)#define NR_FASTOP (ilog2(sizeof(ulong)) + 1) -#define FASTOP_SIZE 8 +#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))struct opcode {u64 flags;
@@ -311,7 +311,8 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);#define __FOP_FUNC (name) \ ".align " __stringify(FASTOP_SIZE) " \n\t" \ ".type " name ", @function \n\t" \ - name ":\n\t" + name ":\n\t" \ + ASM_ENDBR#define FOP_FUNC (name) \ __FOP_FUNC(#name)
@@ -433,6 +434,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);".align 4 \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ + ASM_ENDBR \ #op " %al \n\t" \ __FOP_RET(#op)
[tip: x86/core] x86/ibt,kvm: Add ENDBR to fastops
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 6649fa876da4c505548b8e8945a6fc48e62e427c Gitweb: https://git.kernel.org/tip/6649fa876da4c505548b8e8945a6fc48e62e427c Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:28 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:37 +01:00
x86/ibt,kvm: Add ENDBR to fastops
--- arch/x86/kvm/emulate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
arch/x86/kvm/emulate.c +4 -2
@@ -189,7 +189,7 @@ #define X16(x...) X8(x), X8(x)#define NR_FASTOP (ilog2(sizeof(ulong)) + 1) -#define FASTOP_SIZE 8 +#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))struct opcode {u64 flags;
@@ -311,7 +311,8 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);#define __FOP_FUNC (name) \ ".align " __stringify(FASTOP_SIZE) " \n\t" \ ".type " name ", @function \n\t" \ - name ":\n\t" + name ":\n\t" \ + ASM_ENDBR#define FOP_FUNC (name) \ __FOP_FUNC(#name)
@@ -433,6 +434,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);".align 4 \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ + ASM_ENDBR \ #op " %al \n\t" \ __FOP_RET(#op)
Peter Zijlstra
maintainerPATCH v4 18/45 x86/ibt,ftrace: Search for __fentry__ location
In reply to Peter ZijlstraCurrently a lot of ftrace code assumes __fentry__ is at sym+0. However with Intel IBT enabled the first instruction of a function will most likely be ENDBR.
Change ftrace_location() to not only return the __fentry__ location when called for the __fentry__ location, but also when called for the sym+0 location.
Then audit/update all callsites of this function to consistently use these new semantics.
--- arch/x86/kernel/kprobes/core.c | 11 +--------- kernel/bpf/trampoline.c | 20 +++---------------- kernel/kprobes.c | 8 +------ kernel/trace/ftrace.c | 43 +++++++++++++++++++++++++++++++++-------- 4 files changed, 43 insertions(+), 39 deletions(-)
arch/x86/kernel/kprobes/core.c +2 -9
@@ -193,17 +193,10 @@ static unsigned long __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) {struct kprobe *kp; -unsigned long faddr; +bool faddr; kp =get_kprobe ((void *)addr); - faddr =ftrace_location (addr); -/* -* Addresses inside the ftrace location are refused by -* arch_check_ftrace_location(). Something went terribly wrong -* if such an address is checked here. -*/ -if (WARN_ON (faddr && faddr != addr)) -return 0UL ; + faddr =ftrace_location (addr) == addr;/* * Use the current code if it is not modified by Kprobe * and it cannot be modified by ftrace.
kernel/bpf/trampoline.c +4 -16
@@ -117,18 +117,6 @@ static void bpf_trampoline_module_put(st tr->mod =NULL ; } -static int is_ftrace_location (void *ip) -{ -long addr; - - addr =ftrace_location ((long )ip); -if (!addr) -return 0 ; -if (WARN_ON_ONCE (addr != (long )ip)) -return -EFAULT ; -return 1 ; -} -static int unregister_fentry (struct bpf_trampoline *tr,void *old_addr) {void *ip = tr->func.addr;
@@ -160,12 +148,12 @@ static int modify_fentry(struct bpf_tramstatic int register_fentry (struct bpf_trampoline *tr,void *new_addr) {void *ip = tr->func.addr; +unsigned long faddr;int ret; - ret =is_ftrace_location (ip); -if (ret <0 ) -return ret; - tr->func.ftrace_managed = ret; + faddr =ftrace_location ((unsigned long )ip); +if (faddr) + tr->func.ftrace_managed = true;if (bpf_trampoline_module_get (tr))return -ENOENT ;
kernel/kprobes.c +2 -6
@@ -1562,14 +1562,10 @@ static inline int warn_kprobe_rereg(strustatic int check_ftrace_location (struct kprobe *p) { -unsigned long ftrace_addr; +unsigned long addr = (unsigned long )p->addr; - ftrace_addr =ftrace_location ((unsigned long )p->addr); -if (ftrace_addr) { +if (ftrace_location (addr) == addr) {#ifdef CONFIG_KPROBES_ON_FTRACE -/* Given address is not on the instruction boundary */ -if ((unsigned long )p->addr != ftrace_addr) -return -EILSEQ ; p->flags |=KPROBE_FLAG_FTRACE ;#else /* !CONFIG_KPROBES_ON_FTRACE */ return -EINVAL ;
kernel/trace/ftrace.c +35 -8
76 lines not on this page · raw message
[tip: x86/core] x86/ibt,ftrace: Search for __fentry__ location
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: ee1a8cf8dd0f0b1a2adfb8aedf6c75568a411b0a Gitweb: https://git.kernel.org/tip/ee1a8cf8dd0f0b1a2adfb8aedf6c75568a411b0a Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:29 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:31 +01:00
x86/ibt,ftrace: Search for __fentry__ location
Currently a lot of ftrace code assumes __fentry__ is at sym+0. However with Intel IBT enabled the first instruction of a function will most likely be ENDBR.
Change ftrace_location() to not only return the __fentry__ location when called for the __fentry__ location, but also when called for the sym+0 location.
Then audit/update all callsites of this function to consistently use these new semantics.
--- arch/x86/kernel/kprobes/core.c | 11 +------- kernel/bpf/trampoline.c | 20 +++------------ kernel/kprobes.c | 8 +----- kernel/trace/ftrace.c | 43 ++++++++++++++++++++++++++------- 4 files changed, 43 insertions(+), 39 deletions(-)
arch/x86/kernel/kprobes/core.c +2 -9
@@ -193,17 +193,10 @@ static unsigned long __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) {struct kprobe *kp; -unsigned long faddr; +bool faddr; kp =get_kprobe ((void *)addr); - faddr =ftrace_location (addr); -/* -* Addresses inside the ftrace location are refused by -* arch_check_ftrace_location(). Something went terribly wrong -* if such an address is checked here. -*/ -if (WARN_ON (faddr && faddr != addr)) -return 0UL ; + faddr =ftrace_location (addr) == addr;/* * Use the current code if it is not modified by Kprobe * and it cannot be modified by ftrace.
kernel/bpf/trampoline.c +4 -16
@@ -117,18 +117,6 @@ static void bpf_trampoline_module_put(struct bpf_trampoline *tr) tr->mod =NULL ; } -static int is_ftrace_location (void *ip) -{ -long addr; - - addr =ftrace_location ((long )ip); -if (!addr) -return 0 ; -if (WARN_ON_ONCE (addr != (long )ip)) -return -EFAULT ; -return 1 ; -} -static int unregister_fentry (struct bpf_trampoline *tr,void *old_addr) {void *ip = tr->func.addr;
@@ -160,12 +148,12 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_adstatic int register_fentry (struct bpf_trampoline *tr,void *new_addr) {void *ip = tr->func.addr; +unsigned long faddr;int ret; - ret =is_ftrace_location (ip); -if (ret <0 ) -return ret; - tr->func.ftrace_managed = ret; + faddr =ftrace_location ((unsigned long )ip); +if (faddr) + tr->func.ftrace_managed = true;if (bpf_trampoline_module_get (tr))return -ENOENT ;
kernel/kprobes.c +2 -6
@@ -1562,14 +1562,10 @@ static inline int warn_kprobe_rereg(struct kprobe *p)static int check_ftrace_location (struct kprobe *p) { -unsigned long ftrace_addr; +unsigned long addr = (unsigned long )p->addr; - ftrace_addr =ftrace_location ((unsigned long )p->addr); -if (ftrace_addr) { +if (ftrace_location (addr) == addr) {#ifdef CONFIG_KPROBES_ON_FTRACE -/* Given address is not on the instruction boundary */ -if ((unsigned long )p->addr != ftrace_addr) -return -EILSEQ ; p->flags |=KPROBE_FLAG_FTRACE ;#else /* !CONFIG_KPROBES_ON_FTRACE */ return -EINVAL ;
kernel/trace/ftrace.c +35 -8
76 lines not on this page · raw message
[tip: x86/core] x86/ibt,ftrace: Search for __fentry__ location
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: aebfd12521d9c7d0b502cf6d06314cfbcdccfe3b Gitweb: https://git.kernel.org/tip/aebfd12521d9c7d0b502cf6d06314cfbcdccfe3b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:29 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:37 +01:00
x86/ibt,ftrace: Search for __fentry__ location
Currently a lot of ftrace code assumes __fentry__ is at sym+0. However with Intel IBT enabled the first instruction of a function will most likely be ENDBR.
Change ftrace_location() to not only return the __fentry__ location when called for the __fentry__ location, but also when called for the sym+0 location.
Then audit/update all callsites of this function to consistently use these new semantics.
--- arch/x86/kernel/kprobes/core.c | 11 +------- kernel/bpf/trampoline.c | 20 ++------------ kernel/kprobes.c | 8 +----- kernel/trace/ftrace.c | 48 +++++++++++++++++++++++++++------ 4 files changed, 48 insertions(+), 39 deletions(-)
arch/x86/kernel/kprobes/core.c +2 -9
@@ -193,17 +193,10 @@ static unsigned long __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) {struct kprobe *kp; -unsigned long faddr; +bool faddr; kp =get_kprobe ((void *)addr); - faddr =ftrace_location (addr); -/* -* Addresses inside the ftrace location are refused by -* arch_check_ftrace_location(). Something went terribly wrong -* if such an address is checked here. -*/ -if (WARN_ON (faddr && faddr != addr)) -return 0UL ; + faddr =ftrace_location (addr) == addr;/* * Use the current code if it is not modified by Kprobe * and it cannot be modified by ftrace.
kernel/bpf/trampoline.c +4 -16
@@ -117,18 +117,6 @@ static void bpf_trampoline_module_put(struct bpf_trampoline *tr) tr->mod =NULL ; } -static int is_ftrace_location (void *ip) -{ -long addr; - - addr =ftrace_location ((long )ip); -if (!addr) -return 0 ; -if (WARN_ON_ONCE (addr != (long )ip)) -return -EFAULT ; -return 1 ; -} -static int unregister_fentry (struct bpf_trampoline *tr,void *old_addr) {void *ip = tr->func.addr;
@@ -160,12 +148,12 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_adstatic int register_fentry (struct bpf_trampoline *tr,void *new_addr) {void *ip = tr->func.addr; +unsigned long faddr;int ret; - ret =is_ftrace_location (ip); -if (ret <0 ) -return ret; - tr->func.ftrace_managed = ret; + faddr =ftrace_location ((unsigned long )ip); +if (faddr) + tr->func.ftrace_managed = true;if (bpf_trampoline_module_get (tr))return -ENOENT ;
kernel/kprobes.c +2 -6
@@ -1562,14 +1562,10 @@ static inline int warn_kprobe_rereg(struct kprobe *p)static int check_ftrace_location (struct kprobe *p) { -unsigned long ftrace_addr; +unsigned long addr = (unsigned long )p->addr; - ftrace_addr =ftrace_location ((unsigned long )p->addr); -if (ftrace_addr) { +if (ftrace_location (addr) == addr) {#ifdef CONFIG_KPROBES_ON_FTRACE -/* Given address is not on the instruction boundary */ -if ((unsigned long )p->addr != ftrace_addr) -return -EILSEQ ; p->flags |=KPROBE_FLAG_FTRACE ;#else /* !CONFIG_KPROBES_ON_FTRACE */ return -EINVAL ;
kernel/trace/ftrace.c +40 -8
87 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v4 19/45 x86/livepatch: Validate __fentry__ location
In reply to Peter ZijlstraCurrently livepatch assumes __fentry__ lives at func+0, which is most likely untrue with IBT on. Instead make it use ftrace_location() by default which both validates and finds the actual ip if there is any in the same symbol.
--- arch/powerpc/include/asm/livepatch.h | 10 ---------- kernel/livepatch/patch.c | 19 ++----------------- 2 files changed, 2 insertions(+), 27 deletions(-)
arch/powerpc/include/asm/livepatch.h +0 -10
@@ -19,16 +19,6 @@ static inline void klp_arch_set_pc(strucregs_set_return_ip (regs, ip); } -#define klp_get_ftrace_location klp_get_ftrace_location -static inline unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -/* -* Live patch works only with -mprofile-kernel on PPC. In this case, -* the ftrace location is always within the first 16 bytes. -*/ -return ftrace_location_range (faddr, faddr +16 ); -} -static inline void klp_init_thread_info (struct task_struct *p) {/* + 1 to account for STACK_END_MAGIC */
kernel/livepatch/patch.c +2 -17
@@ -124,19 +124,6 @@ static void notrace klp_ftrace_handler(uftrace_test_recursion_unlock (bit); } -/* -* Convert a function address into the appropriate ftrace location. -* -* Usually this is just the address of the function, but on some architectures -* it's more complicated so allow them to provide a custom behaviour. -*/ -#ifndef klp_get_ftrace_location -static unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -return faddr; -} -#endif -static void klp_unpatch_func (struct klp_func *func) {struct klp_ops *ops;
@@ -153,8 +140,7 @@ static void klp_unpatch_func(struct klp_if (list_is_singular (&ops->func_stack)) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (WARN_ON (!ftrace_loc))return ;
@@ -186,8 +172,7 @@ static int klp_patch_func(struct klp_funif (!ops) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (!ftrace_loc) {pr_err ("failed to find location for function '%s'\n" , func->old_name);
[tip: x86/core] x86/livepatch: Validate __fentry__ location
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: a557abfd1a16072e4d25bdf226e3b761e6f10741 Gitweb: https://git.kernel.org/tip/a557abfd1a16072e4d25bdf226e3b761e6f10741 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:30 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:31 +01:00
x86/livepatch: Validate __fentry__ location
Currently livepatch assumes __fentry__ lives at func+0, which is most likely untrue with IBT on. Instead make it use ftrace_location() by default which both validates and finds the actual ip if there is any in the same symbol.
--- arch/powerpc/include/asm/livepatch.h | 10 ---------- kernel/livepatch/patch.c | 19 ++----------------- 2 files changed, 2 insertions(+), 27 deletions(-)
arch/powerpc/include/asm/livepatch.h +0 -10
@@ -19,16 +19,6 @@ static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip)regs_set_return_ip (regs, ip); } -#define klp_get_ftrace_location klp_get_ftrace_location -static inline unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -/* -* Live patch works only with -mprofile-kernel on PPC. In this case, -* the ftrace location is always within the first 16 bytes. -*/ -return ftrace_location_range (faddr, faddr +16 ); -} -static inline void klp_init_thread_info (struct task_struct *p) {/* + 1 to account for STACK_END_MAGIC */
kernel/livepatch/patch.c +2 -17
@@ -124,19 +124,6 @@ unlock:ftrace_test_recursion_unlock (bit); } -/* -* Convert a function address into the appropriate ftrace location. -* -* Usually this is just the address of the function, but on some architectures -* it's more complicated so allow them to provide a custom behaviour. -*/ -#ifndef klp_get_ftrace_location -static unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -return faddr; -} -#endif -static void klp_unpatch_func (struct klp_func *func) {struct klp_ops *ops;
@@ -153,8 +140,7 @@ static void klp_unpatch_func(struct klp_func *func)if (list_is_singular (&ops->func_stack)) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (WARN_ON (!ftrace_loc))return ;
@@ -186,8 +172,7 @@ static int klp_patch_func(struct klp_func *func)if (!ops) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (!ftrace_loc) {pr_err ("failed to find location for function '%s'\n" , func->old_name);
Re: [PATCH v4 19/45] x86/livepatch: Validate __fentry__ location
In reply to Peter ZijlstraQuoted 7 lines · Peter Zijlstra
Currently livepatch assumes
__fentry__lives at func+0, which is most likely untrue with IBT on. Instead make it useftrace_location()by default which both validates and finds the actual ip if there is any in the same symbol.Suggested-by Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
M
[tip: x86/core] x86/livepatch: Validate __fentry__ location
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: d15cb3dab1e4f00e29599a4f5e1f6678a530d270 Gitweb: https://git.kernel.org/tip/d15cb3dab1e4f00e29599a4f5e1f6678a530d270 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:30 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:37 +01:00
x86/livepatch: Validate __fentry__ location
Currently livepatch assumes __fentry__ lives at func+0, which is most likely untrue with IBT on. Instead make it use ftrace_location() by default which both validates and finds the actual ip if there is any in the same symbol.
--- arch/powerpc/include/asm/livepatch.h | 10 ---------- kernel/livepatch/patch.c | 19 ++----------------- 2 files changed, 2 insertions(+), 27 deletions(-)
arch/powerpc/include/asm/livepatch.h +0 -10
@@ -19,16 +19,6 @@ static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip)regs_set_return_ip (regs, ip); } -#define klp_get_ftrace_location klp_get_ftrace_location -static inline unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -/* -* Live patch works only with -mprofile-kernel on PPC. In this case, -* the ftrace location is always within the first 16 bytes. -*/ -return ftrace_location_range (faddr, faddr +16 ); -} -static inline void klp_init_thread_info (struct task_struct *p) {/* + 1 to account for STACK_END_MAGIC */
kernel/livepatch/patch.c +2 -17
@@ -124,19 +124,6 @@ unlock:ftrace_test_recursion_unlock (bit); } -/* -* Convert a function address into the appropriate ftrace location. -* -* Usually this is just the address of the function, but on some architectures -* it's more complicated so allow them to provide a custom behaviour. -*/ -#ifndef klp_get_ftrace_location -static unsigned long klp_get_ftrace_location (unsigned long faddr) -{ -return faddr; -} -#endif -static void klp_unpatch_func (struct klp_func *func) {struct klp_ops *ops;
@@ -153,8 +140,7 @@ static void klp_unpatch_func(struct klp_func *func)if (list_is_singular (&ops->func_stack)) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (WARN_ON (!ftrace_loc))return ;
@@ -186,8 +172,7 @@ static int klp_patch_func(struct klp_func *func)if (!ops) {unsigned long ftrace_loc; - ftrace_loc = -klp_get_ftrace_location ((unsigned long )func->old_func); + ftrace_loc =ftrace_location ((unsigned long )func->old_func);if (!ftrace_loc) {pr_err ("failed to find location for function '%s'\n" , func->old_name);
Peter Zijlstra
maintainerPATCH v4 20/45 x86/ibt,ftrace: Make function-graph play nice
In reply to Peter ZijlstraReturn trampoline must not use indirect branch to return; while this preserves the RSB, it is fundamentally incompatible with IBT. Instead use a retpoline like ROP gadget that defeats IBT while not unbalancing the RSB.
And since ftrace_stub is no longer a plain RET, don't use it to copy from. Since RET is a trivial instruction, poke it directly.
--- arch/x86/kernel/ftrace.c | 9 ++------- arch/x86/kernel/ftrace_64.S | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-)
arch/x86/kernel/ftrace.c +2 -7
@@ -316,12 +316,12 @@ create_trampoline(struct ftrace_ops *opsunsigned long offset;unsigned long npages;unsigned long size; -unsigned long retq;unsigned long *ptr;void *trampoline;void *ip;/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ unsigned const char op_ref[] = {0x48 ,0x8b ,0x15 }; +unsigned const char retq[] = {RET_INSN_OPCODE ,INT3_INSN_OPCODE };union ftrace_op_code_union op_ptr;int ret;
@@ -359,12 +359,7 @@ create_trampoline(struct ftrace_ops *ops goto fail; ip = trampoline + size; - -/* The trampoline ends with ret(q) */ - retq = (unsigned long )ftrace_stub; - ret =copy_from_kernel_nofault (ip, (void *)retq,RET_SIZE ); -if (WARN_ON (ret <0 )) - goto fail; +memcpy (ip, retq,RET_SIZE );/* No need to test direct calls on created trampolines */ if (ops->flags &FTRACE_OPS_FL_SAVE_REGS ) {
arch/x86/kernel/ftrace_64.S +17 -4
@@ -176,10 +176,10 @@ SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC + ENDBR RET SYM_FUNC_END(ftrace_epilogue)
@@ -284,6 +284,7 @@ SYM_FUNC_START(__fentry__) jnz trace SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) + ENDBR RET trace:
@@ -307,7 +308,7 @@ EXPORT_SYMBOL(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_FUNC_START(return_to_handler) - subq $24, %rsp + subq $16, %rsp /* Save the return values */ movq %rax, (%rsp)
@@ -319,7 +320,19 @@ SYM_FUNC_START(return_to_handler) movq %rax, %rdi movq 8(%rsp), %rdx movq (%rsp), %rax - addq $24, %rsp - JMP_NOSPEC rdi + + addq $16, %rsp + /* + * Jump back to the old return address. This cannot be JMP_NOSPEC rdi + * since IBT would demand that contain ENDBR, which simply isn't so for + * return addresses. Use a retpoline here to keep the RSB balanced. + */ + ANNOTATE_INTRA_FUNCTION_CALL + call .Ldo_rop + int3 +.Ldo_rop: + mov %rdi, (%rsp) + UNWIND_HINT_FUNC + RET SYM_FUNC_END(return_to_handler) #endif
[tip: x86/core] x86/ibt,ftrace: Make function-graph play nice
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 069cfa7285137030111dc78a3fcec091371da514 Gitweb: https://git.kernel.org/tip/069cfa7285137030111dc78a3fcec091371da514 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:31 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:32 +01:00
x86/ibt,ftrace: Make function-graph play nice
Return trampoline must not use indirect branch to return; while this preserves the RSB, it is fundamentally incompatible with IBT. Instead use a retpoline like ROP gadget that defeats IBT while not unbalancing the RSB.
And since ftrace_stub is no longer a plain RET, don't use it to copy from. Since RET is a trivial instruction, poke it directly.
--- arch/x86/kernel/ftrace.c | 9 ++------- arch/x86/kernel/ftrace_64.S | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-)
arch/x86/kernel/ftrace.c +2 -7
@@ -316,12 +316,12 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)unsigned long offset;unsigned long npages;unsigned long size; -unsigned long retq;unsigned long *ptr;void *trampoline;void *ip;/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ unsigned const char op_ref[] = {0x48 ,0x8b ,0x15 }; +unsigned const char retq[] = {RET_INSN_OPCODE ,INT3_INSN_OPCODE };union ftrace_op_code_union op_ptr;int ret;
@@ -359,12 +359,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) goto fail; ip = trampoline + size; - -/* The trampoline ends with ret(q) */ - retq = (unsigned long )ftrace_stub; - ret =copy_from_kernel_nofault (ip, (void *)retq,RET_SIZE ); -if (WARN_ON (ret <0 )) - goto fail; +memcpy (ip, retq,RET_SIZE );/* No need to test direct calls on created trampolines */ if (ops->flags &FTRACE_OPS_FL_SAVE_REGS ) {
arch/x86/kernel/ftrace_64.S +17 -4
@@ -176,10 +176,10 @@ SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC + ENDBR RET SYM_FUNC_END(ftrace_epilogue)
@@ -284,6 +284,7 @@ SYM_FUNC_START(__fentry__) jnz trace SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) + ENDBR RET trace:
@@ -307,7 +308,7 @@ EXPORT_SYMBOL(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_FUNC_START(return_to_handler) - subq $24, %rsp + subq $16, %rsp /* Save the return values */ movq %rax, (%rsp)
@@ -319,7 +320,19 @@ SYM_FUNC_START(return_to_handler) movq %rax, %rdi movq 8(%rsp), %rdx movq (%rsp), %rax - addq $24, %rsp - JMP_NOSPEC rdi + + addq $16, %rsp + /* + * Jump back to the old return address. This cannot be JMP_NOSPEC rdi + * since IBT would demand that contain ENDBR, which simply isn't so for + * return addresses. Use a retpoline here to keep the RSB balanced. + */ + ANNOTATE_INTRA_FUNCTION_CALL + call .Ldo_rop + int3 +.Ldo_rop: + mov %rdi, (%rsp) + UNWIND_HINT_FUNC + RET SYM_FUNC_END(return_to_handler) #endif
[tip: x86/core] x86/ibt,ftrace: Make function-graph play nice
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: e52fc2cf3f662828cc0d51c4b73bed73ad275fce Gitweb: https://git.kernel.org/tip/e52fc2cf3f662828cc0d51c4b73bed73ad275fce Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:31 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:37 +01:00
x86/ibt,ftrace: Make function-graph play nice
Return trampoline must not use indirect branch to return; while this preserves the RSB, it is fundamentally incompatible with IBT. Instead use a retpoline like ROP gadget that defeats IBT while not unbalancing the RSB.
And since ftrace_stub is no longer a plain RET, don't use it to copy from. Since RET is a trivial instruction, poke it directly.
--- arch/x86/kernel/ftrace.c | 9 ++------- arch/x86/kernel/ftrace_64.S | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-)
arch/x86/kernel/ftrace.c +2 -7
@@ -316,12 +316,12 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)unsigned long offset;unsigned long npages;unsigned long size; -unsigned long retq;unsigned long *ptr;void *trampoline;void *ip;/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ unsigned const char op_ref[] = {0x48 ,0x8b ,0x15 }; +unsigned const char retq[] = {RET_INSN_OPCODE ,INT3_INSN_OPCODE };union ftrace_op_code_union op_ptr;int ret;
@@ -359,12 +359,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) goto fail; ip = trampoline + size; - -/* The trampoline ends with ret(q) */ - retq = (unsigned long )ftrace_stub; - ret =copy_from_kernel_nofault (ip, (void *)retq,RET_SIZE ); -if (WARN_ON (ret <0 )) - goto fail; +memcpy (ip, retq,RET_SIZE );/* No need to test direct calls on created trampolines */ if (ops->flags &FTRACE_OPS_FL_SAVE_REGS ) {
arch/x86/kernel/ftrace_64.S +17 -4
@@ -176,10 +176,10 @@ SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC + ENDBR RET SYM_FUNC_END(ftrace_epilogue)
@@ -284,6 +284,7 @@ SYM_FUNC_START(__fentry__) jnz trace SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) + ENDBR RET trace:
@@ -307,7 +308,7 @@ EXPORT_SYMBOL(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_FUNC_START(return_to_handler) - subq $24, %rsp + subq $16, %rsp /* Save the return values */ movq %rax, (%rsp)
@@ -319,7 +320,19 @@ SYM_FUNC_START(return_to_handler) movq %rax, %rdi movq 8(%rsp), %rdx movq (%rsp), %rax - addq $24, %rsp - JMP_NOSPEC rdi + + addq $16, %rsp + /* + * Jump back to the old return address. This cannot be JMP_NOSPEC rdi + * since IBT would demand that contain ENDBR, which simply isn't so for + * return addresses. Use a retpoline here to keep the RSB balanced. + */ + ANNOTATE_INTRA_FUNCTION_CALL + call .Ldo_rop + int3 +.Ldo_rop: + mov %rdi, (%rsp) + UNWIND_HINT_FUNC + RET SYM_FUNC_END(return_to_handler) #endif
Peter Zijlstra
maintainerPATCH v4 21/45 x86/ibt,kprobes: Cure sym+0 equals fentry woes
In reply to Peter ZijlstraIn order to allow kprobes to skip the ENDBR instructions at sym+0 for X86_KERNEL_IBT builds, change _kprobe_addr() to take an architecture callback to inspect the function at hand and modify the offset if needed.
This streamlines the existing interface to cover more cases and require less hooks. Once PowerPC gets fully converted there will only be the one arch hook.
--- arch/powerpc/kernel/kprobes.c | 34 +++++++++++++-------- arch/x86/kernel/kprobes/core.c | 28 ++++++++++++++--- include/linux/kprobes.h | 3 + kernel/kprobes.c | 66 ++++++++++++++++++++++++++++++++--------- 4 files changed, 98 insertions(+), 33 deletions(-)
arch/powerpc/kernel/kprobes.c +21 -13
@@ -105,6 +105,27 @@ kprobe_opcode_t *kprobe_lookup_name(consreturn addr; } +static bool arch_kprobe_on_func_entry (unsigned long offset) +{ +#ifdef PPC64_ELF_ABI_v2 +#ifdef CONFIG_KPROBES_ON_FTRACE +return offset <=16 ; +#else +return offset <=8 ; +#endif +#else +return !offset; +#endif +} + +/* XXX try and fold the magic of kprobe_lookup_name() in this */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ + *on_func_entry =arch_kprobe_on_func_entry (offset); +return (kprobe_opcode_t *)(addr + offset); +} +void *alloc_insn_page (void ) {void *page;
@@ -218,19 +239,6 @@ static nokprobe_inline void set_current_ kcb->kprobe_saved_msr = regs->msr; } -bool arch_kprobe_on_func_entry (unsigned long offset) -{ -#ifdef PPC64_ELF_ABI_v2 -#ifdef CONFIG_KPROBES_ON_FTRACE -return offset <=16 ; -#else -return offset <=8 ; -#endif -#else -return !offset; -#endif -} -void arch_prepare_kretprobe (struct kretprobe_instance *ri,struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)regs->link;
arch/x86/kernel/kprobes/core.c +17 -0
@@ -52,6 +52,7 @@#include <asm/insn.h> #include <asm/debugreg.h> #include <asm/set_memory.h> +#include <asm/ibt.h> #include "common.h"
@@ -301,6 +303,22 @@ static int can_probe(unsigned long paddrreturn (addr == paddr); } +/* If x86 supports IBT (ENDBR) it must be skipped. */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ +if (is_endbr (*(u32 *)addr)) { + *on_func_entry = !offset || offset ==4 ; +if (*on_func_entry) + offset =4 ; + + }else { + *on_func_entry = !offset; + } + +return (kprobe_opcode_t *)(addr + offset); +} +/* * Copy an instruction with recovering modified instruction by kprobes * and adjust the displacement if the instruction uses the %rip-relative
include/linux/kprobes.h +2 -1
@@ -265,7 +265,6 @@ extern int arch_init_kprobes(void);extern void kprobes_inc_nmissed_count (struct kprobe *p);extern bool arch_within_kprobe_blacklist (unsigned long addr);extern int arch_populate_kprobe_blacklist (void ); -extern bool arch_kprobe_on_func_entry (unsigned long offset);extern int kprobe_on_func_entry (kprobe_opcode_t *addr,const char *sym,unsigned long offset);extern bool within_kprobe_blacklist (unsigned long addr);
@@ -384,6 +383,8 @@ static inline struct kprobe_ctlblk *get_ }kprobe_opcode_t *kprobe_lookup_name (const char *name,unsigned int offset); +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset,bool *on_func_entry); +int register_kprobe (struct kprobe *p);void unregister_kprobe (struct kprobe *p);int register_kprobes (struct kprobe **kps,int num);
kernel/kprobes.c +52 -14
109 lines not on this page · raw message
[tip: x86/core] x86/ibt,kprobes: Cure sym+0 equals fentry woes
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 19d1033ec715d873e490841b07f300bb16cdcd21 Gitweb: https://git.kernel.org/tip/19d1033ec715d873e490841b07f300bb16cdcd21 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:32 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:32 +01:00
x86/ibt,kprobes: Cure sym+0 equals fentry woes
In order to allow kprobes to skip the ENDBR instructions at sym+0 for X86_KERNEL_IBT builds, change _kprobe_addr() to take an architecture callback to inspect the function at hand and modify the offset if needed.
This streamlines the existing interface to cover more cases and require less hooks. Once PowerPC gets fully converted there will only be the one arch hook.
--- arch/powerpc/kernel/kprobes.c | 34 ++++++++++------- arch/x86/kernel/kprobes/core.c | 17 +++++++++- include/linux/kprobes.h | 3 +- kernel/kprobes.c | 66 ++++++++++++++++++++++++++------- 4 files changed, 92 insertions(+), 28 deletions(-)
arch/powerpc/kernel/kprobes.c +21 -13
@@ -105,6 +105,27 @@ kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)return addr; } +static bool arch_kprobe_on_func_entry (unsigned long offset) +{ +#ifdef PPC64_ELF_ABI_v2 +#ifdef CONFIG_KPROBES_ON_FTRACE +return offset <=16 ; +#else +return offset <=8 ; +#endif +#else +return !offset; +#endif +} + +/* XXX try and fold the magic of kprobe_lookup_name() in this */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ + *on_func_entry =arch_kprobe_on_func_entry (offset); +return (kprobe_opcode_t *)(addr + offset); +} +void *alloc_insn_page (void ) {void *page;
@@ -218,19 +239,6 @@ static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs kcb->kprobe_saved_msr = regs->msr; } -bool arch_kprobe_on_func_entry (unsigned long offset) -{ -#ifdef PPC64_ELF_ABI_v2 -#ifdef CONFIG_KPROBES_ON_FTRACE -return offset <=16 ; -#else -return offset <=8 ; -#endif -#else -return !offset; -#endif -} -void arch_prepare_kretprobe (struct kretprobe_instance *ri,struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)regs->link;
arch/x86/kernel/kprobes/core.c +17 -0
@@ -52,6 +52,7 @@#include <asm/insn.h> #include <asm/debugreg.h> #include <asm/set_memory.h> +#include <asm/ibt.h> #include "common.h"
@@ -294,6 +295,22 @@ static int can_probe(unsigned long paddr)return (addr == paddr); } +/* If x86 supports IBT (ENDBR) it must be skipped. */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ +if (is_endbr (*(u32 *)addr)) { + *on_func_entry = !offset || offset ==4 ; +if (*on_func_entry) + offset =4 ; + + }else { + *on_func_entry = !offset; + } + +return (kprobe_opcode_t *)(addr + offset); +} +/* * Copy an instruction with recovering modified instruction by kprobes * and adjust the displacement if the instruction uses the %rip-relative
include/linux/kprobes.h +2 -1
@@ -265,7 +265,6 @@ extern int arch_init_kprobes(void);extern void kprobes_inc_nmissed_count (struct kprobe *p);extern bool arch_within_kprobe_blacklist (unsigned long addr);extern int arch_populate_kprobe_blacklist (void ); -extern bool arch_kprobe_on_func_entry (unsigned long offset);extern int kprobe_on_func_entry (kprobe_opcode_t *addr,const char *sym,unsigned long offset);extern bool within_kprobe_blacklist (unsigned long addr);
@@ -384,6 +383,8 @@ static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) }kprobe_opcode_t *kprobe_lookup_name (const char *name,unsigned int offset); +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset,bool *on_func_entry); +int register_kprobe (struct kprobe *p);void unregister_kprobe (struct kprobe *p);int register_kprobes (struct kprobe **kps,int num);
kernel/kprobes.c +52 -14
109 lines not on this page · raw message
[tip: x86/core] x86/ibt,kprobes: Cure sym+0 equals fentry woes
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: cc66bb91457827f62e2b6cb2518666820f0a6c48 Gitweb: https://git.kernel.org/tip/cc66bb91457827f62e2b6cb2518666820f0a6c48 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:32 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:38 +01:00
x86/ibt,kprobes: Cure sym+0 equals fentry woes
In order to allow kprobes to skip the ENDBR instructions at sym+0 for X86_KERNEL_IBT builds, change _kprobe_addr() to take an architecture callback to inspect the function at hand and modify the offset if needed.
This streamlines the existing interface to cover more cases and require less hooks. Once PowerPC gets fully converted there will only be the one arch hook.
--- arch/powerpc/kernel/kprobes.c | 34 ++++++++++------- arch/x86/kernel/kprobes/core.c | 17 +++++++++- include/linux/kprobes.h | 3 +- kernel/kprobes.c | 66 ++++++++++++++++++++++++++------- 4 files changed, 92 insertions(+), 28 deletions(-)
arch/powerpc/kernel/kprobes.c +21 -13
@@ -105,6 +105,27 @@ kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)return addr; } +static bool arch_kprobe_on_func_entry (unsigned long offset) +{ +#ifdef PPC64_ELF_ABI_v2 +#ifdef CONFIG_KPROBES_ON_FTRACE +return offset <=16 ; +#else +return offset <=8 ; +#endif +#else +return !offset; +#endif +} + +/* XXX try and fold the magic of kprobe_lookup_name() in this */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ + *on_func_entry =arch_kprobe_on_func_entry (offset); +return (kprobe_opcode_t *)(addr + offset); +} +void *alloc_insn_page (void ) {void *page;
@@ -218,19 +239,6 @@ static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs kcb->kprobe_saved_msr = regs->msr; } -bool arch_kprobe_on_func_entry (unsigned long offset) -{ -#ifdef PPC64_ELF_ABI_v2 -#ifdef CONFIG_KPROBES_ON_FTRACE -return offset <=16 ; -#else -return offset <=8 ; -#endif -#else -return !offset; -#endif -} -void arch_prepare_kretprobe (struct kretprobe_instance *ri,struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)regs->link;
arch/x86/kernel/kprobes/core.c +17 -0
@@ -52,6 +52,7 @@#include <asm/insn.h> #include <asm/debugreg.h> #include <asm/set_memory.h> +#include <asm/ibt.h> #include "common.h"
@@ -294,6 +295,22 @@ static int can_probe(unsigned long paddr)return (addr == paddr); } +/* If x86 supports IBT (ENDBR) it must be skipped. */ +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset, +bool *on_func_entry) +{ +if (is_endbr (*(u32 *)addr)) { + *on_func_entry = !offset || offset ==4 ; +if (*on_func_entry) + offset =4 ; + + }else { + *on_func_entry = !offset; + } + +return (kprobe_opcode_t *)(addr + offset); +} +/* * Copy an instruction with recovering modified instruction by kprobes * and adjust the displacement if the instruction uses the %rip-relative
include/linux/kprobes.h +2 -1
@@ -265,7 +265,6 @@ extern int arch_init_kprobes(void);extern void kprobes_inc_nmissed_count (struct kprobe *p);extern bool arch_within_kprobe_blacklist (unsigned long addr);extern int arch_populate_kprobe_blacklist (void ); -extern bool arch_kprobe_on_func_entry (unsigned long offset);extern int kprobe_on_func_entry (kprobe_opcode_t *addr,const char *sym,unsigned long offset);extern bool within_kprobe_blacklist (unsigned long addr);
@@ -384,6 +383,8 @@ static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) }kprobe_opcode_t *kprobe_lookup_name (const char *name,unsigned int offset); +kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset,bool *on_func_entry); +int register_kprobe (struct kprobe *p);void unregister_kprobe (struct kprobe *p);int register_kprobes (struct kprobe **kps,int num);
kernel/kprobes.c +52 -14
109 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v4 22/45 x86/ibt,bpf: Add ENDBR instructions to prologue and trampoline
In reply to Peter ZijlstraWith IBT enabled builds we need ENDBR instructions at indirect jump target sites, since we start execution of the JIT'ed code through an indirect jump, the very first instruction needs to be ENDBR.
Similarly, since eBPF tail-calls use indirect branches, their landing site needs to be an ENDBR too.
The trampolines need similar adjustment.
--- arch/x86/net/bpf_jit_comp.c | 16 ++++++++++++++-- kernel/bpf/trampoline.c | 20 ++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-)
arch/x86/net/bpf_jit_comp.c +14 -2
@@ -46,6 +46,12 @@ static u8 *emit_code(u8 *ptr, u32 bytes,#define EMIT4_off32 (b1, b2, b3, b4, off) \ do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) +#ifdef CONFIG_X86_KERNEL_IBT +#define EMIT_ENDBR () EMIT(gen_endbr(), 4) +#else +#define EMIT_ENDBR () +#endif +static bool is_imm8 (int value) {return value <=127 && value >=-128 ;
@@ -241,7 +247,7 @@ struct jit_context {/* Number of bytes emit_patch() needs to generate instructions */ #define X86_PATCH_SIZE 5/* Number of bytes that will be skipped on tailcall */ -#define X86_TAIL_CALL_OFFSET 11 +#define X86_TAIL_CALL_OFFSET (11 + ENDBR_INSN_SIZE)static void push_callee_regs (u8 **pprog,bool *callee_regs_used) {
@@ -286,6 +292,7 @@ static void emit_prologue(u8 **pprog, u3/* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ +EMIT_ENDBR ();memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ;if (!ebpf_from_cbpf) {
@@ -296,6 +303,10 @@ static void emit_prologue(u8 **pprog, u3 }EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ + +/* X86_TAIL_CALL_OFFSET is here */ +EMIT_ENDBR (); +/* sub rsp, rounded_stack_depth */ if (stack_depth)EMIT3_off32 (0x48 ,0x81 ,0xEC ,round_up (stack_depth,8 ));
@@ -2028,10 +2039,11 @@ int arch_prepare_bpf_trampoline(struct b/* skip patched call instruction and point orig_call to actual * body of the kernel function. */ - orig_call +=X86_PATCH_SIZE ; + orig_call +=X86_PATCH_SIZE +ENDBR_INSN_SIZE ; prog = image; +EMIT_ENDBR ();EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ EMIT4 (0x48 ,0x83 ,0xEC , stack_size);/* sub rsp, stack_size */
[tip: x86/core] x86/ibt,bpf: Add ENDBR instructions to prologue and trampoline
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2f6f7bf13ab67330390bb19afe605dc2abfe726b Gitweb: https://git.kernel.org/tip/2f6f7bf13ab67330390bb19afe605dc2abfe726b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:33 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:32 +01:00
x86/ibt,bpf: Add ENDBR instructions to prologue and trampoline
With IBT enabled builds we need ENDBR instructions at indirect jump target sites, since we start execution of the JIT'ed code through an indirect jump, the very first instruction needs to be ENDBR.
Similarly, since eBPF tail-calls use indirect branches, their landing site needs to be an ENDBR too.
The trampolines need similar adjustment.
--- arch/x86/net/bpf_jit_comp.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-)
arch/x86/net/bpf_jit_comp.c +14 -2
@@ -46,6 +46,12 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)#define EMIT4_off32 (b1, b2, b3, b4, off) \ do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) +#ifdef CONFIG_X86_KERNEL_IBT +#define EMIT_ENDBR () EMIT(gen_endbr(), 4) +#else +#define EMIT_ENDBR () +#endif +static bool is_imm8 (int value) {return value <=127 && value >=-128 ;
@@ -241,7 +247,7 @@ struct jit_context {/* Number of bytes emit_patch() needs to generate instructions */ #define X86_PATCH_SIZE 5/* Number of bytes that will be skipped on tailcall */ -#define X86_TAIL_CALL_OFFSET 11 +#define X86_TAIL_CALL_OFFSET (11 + ENDBR_INSN_SIZE)static void push_callee_regs (u8 **pprog,bool *callee_regs_used) {
@@ -286,6 +292,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,/* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ +EMIT_ENDBR ();memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ;if (!ebpf_from_cbpf) {
@@ -296,6 +303,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, }EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ + +/* X86_TAIL_CALL_OFFSET is here */ +EMIT_ENDBR (); +/* sub rsp, rounded_stack_depth */ if (stack_depth)EMIT3_off32 (0x48 ,0x81 ,0xEC ,round_up (stack_depth,8 ));
@@ -2028,10 +2039,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i/* skip patched call instruction and point orig_call to actual * body of the kernel function. */ - orig_call +=X86_PATCH_SIZE ; + orig_call +=X86_PATCH_SIZE +ENDBR_INSN_SIZE ; prog = image; +EMIT_ENDBR ();EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ EMIT4 (0x48 ,0x83 ,0xEC , stack_size);/* sub rsp, stack_size */
[tip: x86/core] x86/ibt,bpf: Add ENDBR instructions to prologue and trampoline
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 58912710558889629daae3e0824daacab663bd4a Gitweb: https://git.kernel.org/tip/58912710558889629daae3e0824daacab663bd4a Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:33 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:38 +01:00
x86/ibt,bpf: Add ENDBR instructions to prologue and trampoline
With IBT enabled builds we need ENDBR instructions at indirect jump target sites, since we start execution of the JIT'ed code through an indirect jump, the very first instruction needs to be ENDBR.
Similarly, since eBPF tail-calls use indirect branches, their landing site needs to be an ENDBR too.
The trampolines need similar adjustment.
--- arch/x86/net/bpf_jit_comp.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-)
arch/x86/net/bpf_jit_comp.c +24 -2
@@ -46,6 +46,12 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)#define EMIT4_off32 (b1, b2, b3, b4, off) \ do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) +#ifdef CONFIG_X86_KERNEL_IBT +#define EMIT_ENDBR () EMIT(gen_endbr(), 4) +#else +#define EMIT_ENDBR () +#endif +static bool is_imm8 (int value) {return value <=127 && value >=-128 ;
@@ -241,7 +247,7 @@ struct jit_context {/* Number of bytes emit_patch() needs to generate instructions */ #define X86_PATCH_SIZE 5/* Number of bytes that will be skipped on tailcall */ -#define X86_TAIL_CALL_OFFSET 11 +#define X86_TAIL_CALL_OFFSET (11 + ENDBR_INSN_SIZE)static void push_callee_regs (u8 **pprog,bool *callee_regs_used) {
@@ -286,6 +292,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,/* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ +EMIT_ENDBR ();memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ;if (!ebpf_from_cbpf) {
@@ -296,6 +303,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, }EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ + +/* X86_TAIL_CALL_OFFSET is here */ +EMIT_ENDBR (); +/* sub rsp, rounded_stack_depth */ if (stack_depth)EMIT3_off32 (0x48 ,0x81 ,0xEC ,round_up (stack_depth,8 ));
@@ -384,6 +395,13 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,/* BPF poking in modules is not supported */ return -EINVAL ; +/* +* See emit_prologue(), for IBT builds the trampoline hook is preceded +* with an ENDBR instruction. +*/ +if (is_endbr (*(u32 *)ip)) + ip +=ENDBR_INSN_SIZE ; +return __bpf_arch_text_poke (ip, t, old_addr, new_addr, true); }
@@ -2024,14 +2042,18 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ip_off = stack_size; -if (flags &BPF_TRAMP_F_SKIP_FRAME ) +if (flags &BPF_TRAMP_F_SKIP_FRAME ) {/* skip patched call instruction and point orig_call to actual * body of the kernel function. */ +if (is_endbr (*(u32 *)orig_call)) + orig_call +=ENDBR_INSN_SIZE ; orig_call +=X86_PATCH_SIZE ; + } prog = image; +EMIT_ENDBR ();EMIT1 (0x55 );/* push rbp */ EMIT3 (0x48 ,0x89 ,0xE5 );/* mov rbp, rsp */ EMIT4 (0x48 ,0x83 ,0xEC , stack_size);/* sub rsp, stack_size */
Peter Zijlstra
maintainerPATCH v4 23/45 x86/ibt,ftrace: Add ENDBR to samples/ftrace
In reply to Peter Zijlstra--- samples/ftrace/ftrace-direct-modify.c | 5 +++++ samples/ftrace/ftrace-direct-multi-modify.c | 10 +++++++--- samples/ftrace/ftrace-direct-multi.c | 5 ++++- samples/ftrace/ftrace-direct-too.c | 3 +++ samples/ftrace/ftrace-direct.c | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-)
samples/ftrace/ftrace-direct-modify.c +5 -0
@@ -24,20 +24,25 @@ static unsigned long my_ip = (unsigned l#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func1\n" " leave\n" " .size my_tramp1, .-my_tramp1\n" ASM_RET +" .type my_tramp2, @function\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func2\n"
samples/ftrace/ftrace-direct-multi-modify.c +7 -3
@@ -22,11 +22,14 @@ extern void my_tramp2(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -34,12 +37,13 @@ asm (" call my_direct_func1\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp1, .-my_tramp1\n" +" .type my_tramp2, @function\n" -"\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -47,7 +51,7 @@ asm (" call my_direct_func2\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp2, .-my_tramp2\n" " .popsection\n" );
samples/ftrace/ftrace-direct-multi.c +4 -1
@@ -17,11 +17,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -29,7 +32,7 @@ asm (" call my_direct_func\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
samples/ftrace/ftrace-direct-too.c +3 -0
@@ -19,11 +19,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
samples/ftrace/ftrace-direct.c +3 -0
@@ -16,11 +16,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
[tip: x86/core] x86/ibt,ftrace: Add ENDBR to samples/ftrace
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: cba7a74a1b138c2b7379c88fbbe7c6b94d5bd180 Gitweb: https://git.kernel.org/tip/cba7a74a1b138c2b7379c88fbbe7c6b94d5bd180 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:34 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:33 +01:00
x86/ibt,ftrace: Add ENDBR to samples/ftrace
--- samples/ftrace/ftrace-direct-modify.c | 5 +++++ samples/ftrace/ftrace-direct-multi-modify.c | 10 +++++++--- samples/ftrace/ftrace-direct-multi.c | 5 ++++- samples/ftrace/ftrace-direct-too.c | 3 +++ samples/ftrace/ftrace-direct.c | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-)
samples/ftrace/ftrace-direct-modify.c +5 -0
@@ -24,20 +24,25 @@ static unsigned long my_ip = (unsigned long)schedule;#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func1\n" " leave\n" " .size my_tramp1, .-my_tramp1\n" ASM_RET +" .type my_tramp2, @function\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func2\n"
samples/ftrace/ftrace-direct-multi-modify.c +7 -3
@@ -22,11 +22,14 @@ extern void my_tramp2(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -34,12 +37,13 @@ asm (" call my_direct_func1\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp1, .-my_tramp1\n" +" .type my_tramp2, @function\n" -"\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -47,7 +51,7 @@ asm (" call my_direct_func2\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp2, .-my_tramp2\n" " .popsection\n" );
samples/ftrace/ftrace-direct-multi.c +4 -1
@@ -17,11 +17,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -29,7 +32,7 @@ asm (" call my_direct_func\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
samples/ftrace/ftrace-direct-too.c +3 -0
@@ -19,11 +19,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
samples/ftrace/ftrace-direct.c +3 -0
@@ -16,11 +16,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
[tip: x86/core] x86/ibt,ftrace: Add ENDBR to samples/ftrace
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 0aec21cfb51bc1856206f312d8c13bf1f368d78e Gitweb: https://git.kernel.org/tip/0aec21cfb51bc1856206f312d8c13bf1f368d78e Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:34 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:38 +01:00
x86/ibt,ftrace: Add ENDBR to samples/ftrace
--- samples/ftrace/ftrace-direct-modify.c | 5 +++++ samples/ftrace/ftrace-direct-multi-modify.c | 10 +++++++--- samples/ftrace/ftrace-direct-multi.c | 5 ++++- samples/ftrace/ftrace-direct-too.c | 3 +++ samples/ftrace/ftrace-direct.c | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-)
samples/ftrace/ftrace-direct-modify.c +5 -0
@@ -24,20 +24,25 @@ static unsigned long my_ip = (unsigned long)schedule;#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func1\n" " leave\n" " .size my_tramp1, .-my_tramp1\n" ASM_RET +" .type my_tramp2, @function\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " call my_direct_func2\n"
samples/ftrace/ftrace-direct-multi-modify.c +7 -3
@@ -22,11 +22,14 @@ extern void my_tramp2(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp1, @function\n" " .globl my_tramp1\n" " my_tramp1:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -34,12 +37,13 @@ asm (" call my_direct_func1\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp1, .-my_tramp1\n" +" .type my_tramp2, @function\n" -"\n" " .globl my_tramp2\n" " my_tramp2:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -47,7 +51,7 @@ asm (" call my_direct_func2\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp2, .-my_tramp2\n" " .popsection\n" );
samples/ftrace/ftrace-direct-multi.c +4 -1
@@ -17,11 +17,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
@@ -29,7 +32,7 @@ asm (" call my_direct_func\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
samples/ftrace/ftrace-direct-too.c +3 -0
@@ -19,11 +19,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
samples/ftrace/ftrace-direct.c +3 -0
@@ -16,11 +16,14 @@ extern void my_tramp(void *);#ifdef CONFIG_X86_64 +#include <asm/ibt.h> + asm (" .pushsection .text, \"ax\", @progbits\n" " .type my_tramp, @function\n" " .globl my_tramp\n" " my_tramp:" +ASM_ENDBR " pushq %rbp\n" " movq %rsp, %rbp\n" " pushq %rdi\n"
Peter Zijlstra
maintainerPATCH v4 24/45 x86/ibt: Add IBT feature, MSR and #CP handling
In reply to Peter ZijlstraThe bits required to make the hardware go.. Of note is that, provided the syscall entry points are covered with ENDBR, #CP doesn't need to be an IST because we'll never hit the syscall gap.
--- arch/x86/include/asm/cpu.h | 1 arch/x86/include/asm/cpufeatures.h | 1 arch/x86/include/asm/idtentry.h | 5 + arch/x86/include/asm/msr-index.h | 20 +++++++ arch/x86/include/asm/traps.h | 2 arch/x86/include/uapi/asm/processor-flags.h | 2 arch/x86/kernel/cpu/common.c | 25 +++++++++ arch/x86/kernel/idt.c | 4 + arch/x86/kernel/traps.c | 74 ++++++++++++++++++++++++++++ 9 files changed, 132 insertions(+), 2 deletions(-)
arch/x86/include/asm/cpu.h +1 -0
@@ -7,6 +7,7 @@#include <linux/topology.h> #include <linux/nodemask.h> #include <linux/percpu.h> +#include <asm/ibt.h> #ifdef CONFIG_SMP
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -387,6 +387,7 @@#define X86_FEATURE_TSXLDTRK (18*32+16)/* TSX Suspend Load Address Tracking */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_AMX_BF16 (18*32+22)/* AMX bf16 Support */ #define X86_FEATURE_AVX512_FP16 (18*32+23)/* AVX512 FP16 */ #define X86_FEATURE_AMX_TILE (18*32+24)/* AMX tile Support */
arch/x86/include/asm/idtentry.h +5 -0
@@ -617,6 +617,11 @@ DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_douDECLARE_IDTENTRY_RAW_ERRORCODE (X86_TRAP_DF , xenpv_exc_double_fault);#endif +/* #CP */ +#ifdef CONFIG_X86_KERNEL_IBT +DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_CP , exc_control_protection); +#endif +/* #VC */ #ifdef CONFIG_AMD_MEM_ENCRYPT DECLARE_IDTENTRY_VC (X86_TRAP_VC , exc_vmm_communication);
arch/x86/include/asm/msr-index.h +19 -1
@@ -362,11 +362,29 @@#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d -#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x000006a0/* user mode cet */ +#define MSR_IA32_S_CET 0x000006a2/* kernel mode cet */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) + +#define MSR_IA32_PL0_SSP 0x000006a4/* ring-0 shadow stack pointer */ +#define MSR_IA32_PL1_SSP 0x000006a5/* ring-1 shadow stack pointer */ +#define MSR_IA32_PL2_SSP 0x000006a6/* ring-2 shadow stack pointer */ +#define MSR_IA32_PL3_SSP 0x000006a7/* ring-3 shadow stack pointer */ +#define MSR_IA32_INT_SSP_TAB 0x000006a8/* exception shadow stack table */ +/* Hardware P state interface */ #define MSR_PPERF 0x0000064e#define MSR_PERF_LIMIT_REASONS 0x0000064f
arch/x86/include/asm/traps.h +2 -0
@@ -18,6 +18,8 @@ void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);#endif +extern bool ibt_selftest (void ); +#ifdef CONFIG_X86_F00F_BUG /* For handling the FOOF bug */ void handle_invalid_op (struct pt_regs *regs);
arch/x86/include/uapi/asm/processor-flags.h +2 -0
@@ -130,6 +130,8 @@#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)#define X86_CR4_PKE_BIT 22/* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_CET_BIT 23/* enable Control-flow Enforcement Technology */ +#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)/* * x86-64 Task Priority Register, CR8
arch/x86/kernel/cpu/common.c +24 -1
@@ -59,6 +59,7 @@#include <asm/cpu_device_id.h> #include <asm/uv/uv.h> #include <asm/sigframe.h> +#include <asm/traps.h> #include "cpu.h"
@@ -438,7 +439,8 @@ static __always_inline void setup_umip(s/* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = -X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP |X86_CR4_FSGSBASE ; +X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP | +X86_CR4_FSGSBASE |X86_CR4_CET ;static DEFINE_STATIC_KEY_FALSE_RO (cr_pinning);static unsigned long cr4_pinned_bits__ro_after_init ;
@@ -592,6 +594,24 @@ static __init int setup_disable_pku(char__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +static __always_inline void setup_cet(struct cpuinfo_x86 *c) +{ +u64 msr =CET_ENDBR_EN ; + +if (!HAS_KERNEL_IBT || + !cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +wrmsrl (MSR_IA32_S_CET , msr); +cr4_set_bits (X86_CR4_CET ); + +if (!ibt_selftest ()) { +pr_err ("IBT selftest: Failed!\n" ); +setup_clear_cpu_cap (X86_FEATURE_IBT ); +return ; + } +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
@@ -1709,6 +1729,7 @@ static void identify_cpu(struct cpuinfo_x86_init_rdrand (c);setup_pku (c); +setup_cet (c);/* * Clear/Set all flags overridden by options, need do it
@@ -1777,6 +1798,8 @@ void enable_sep_cpu(void)void __init identify_boot_cpu (void ) {identify_cpu (&boot_cpu_data); +if (HAS_KERNEL_IBT &&cpu_feature_enabled (X86_FEATURE_IBT )) +pr_info ("CET detected: Indirect Branch Tracking enabled\n" );#ifdef CONFIG_X86_32 sysenter_setup ();enable_sep_cpu ();
arch/x86/kernel/idt.c +4 -0
@@ -104,6 +104,10 @@ static const __initconst struct idt_data ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),#endif +#ifdef CONFIG_X86_KERNEL_IBT + INTG(X86_TRAP_CP, asm_exc_control_protection), +#endif +#ifdef CONFIG_AMD_MEM_ENCRYPT ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),#endif
arch/x86/kernel/traps.c +74 -0
80 lines not on this page · raw message
[tip: x86/core] x86/ibt: Add IBT feature, MSR and #CP handling
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 103c0093ceb68e8fde55b809c6b5fb7fbcd60267 Gitweb: https://git.kernel.org/tip/103c0093ceb68e8fde55b809c6b5fb7fbcd60267 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:35 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:33 +01:00
x86/ibt: Add IBT feature, MSR and #CP handling
The bits required to make the hardware go.. Of note is that, provided the syscall entry points are covered with ENDBR, #CP doesn't need to be an IST because we'll never hit the syscall gap.
--- arch/x86/include/asm/cpu.h | 1 +- arch/x86/include/asm/cpufeatures.h | 1 +- arch/x86/include/asm/idtentry.h | 5 +- arch/x86/include/asm/msr-index.h | 20 ++++- arch/x86/include/asm/traps.h | 2 +- arch/x86/include/uapi/asm/processor-flags.h | 2 +- arch/x86/kernel/cpu/common.c | 25 ++++++- arch/x86/kernel/idt.c | 4 +- arch/x86/kernel/traps.c | 74 ++++++++++++++++++++- 9 files changed, 132 insertions(+), 2 deletions(-)
arch/x86/include/asm/cpu.h +1 -0
@@ -7,6 +7,7 @@#include <linux/topology.h> #include <linux/nodemask.h> #include <linux/percpu.h> +#include <asm/ibt.h> #ifdef CONFIG_SMP
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -390,6 +390,7 @@#define X86_FEATURE_TSXLDTRK (18*32+16)/* TSX Suspend Load Address Tracking */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_AVX512_FP16 (18*32+23)/* AVX512 FP16 */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */
arch/x86/include/asm/idtentry.h +5 -0
@@ -617,6 +617,11 @@ DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);DECLARE_IDTENTRY_RAW_ERRORCODE (X86_TRAP_DF , xenpv_exc_double_fault);#endif +/* #CP */ +#ifdef CONFIG_X86_KERNEL_IBT +DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_CP , exc_control_protection); +#endif +/* #VC */ #ifdef CONFIG_AMD_MEM_ENCRYPT DECLARE_IDTENTRY_VC (X86_TRAP_VC , exc_vmm_communication);
arch/x86/include/asm/msr-index.h +19 -1
@@ -360,11 +360,29 @@#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d -#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x000006a0/* user mode cet */ +#define MSR_IA32_S_CET 0x000006a2/* kernel mode cet */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) + +#define MSR_IA32_PL0_SSP 0x000006a4/* ring-0 shadow stack pointer */ +#define MSR_IA32_PL1_SSP 0x000006a5/* ring-1 shadow stack pointer */ +#define MSR_IA32_PL2_SSP 0x000006a6/* ring-2 shadow stack pointer */ +#define MSR_IA32_PL3_SSP 0x000006a7/* ring-3 shadow stack pointer */ +#define MSR_IA32_INT_SSP_TAB 0x000006a8/* exception shadow stack table */ +/* Hardware P state interface */ #define MSR_PPERF 0x0000064e#define MSR_PERF_LIMIT_REASONS 0x0000064f
arch/x86/include/asm/traps.h +2 -0
@@ -18,6 +18,8 @@ void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);#endif +extern bool ibt_selftest (void ); +#ifdef CONFIG_X86_F00F_BUG /* For handling the FOOF bug */ void handle_invalid_op (struct pt_regs *regs);
arch/x86/include/uapi/asm/processor-flags.h +2 -0
@@ -130,6 +130,8 @@#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)#define X86_CR4_PKE_BIT 22/* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_CET_BIT 23/* enable Control-flow Enforcement Technology */ +#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)/* * x86-64 Task Priority Register, CR8
arch/x86/kernel/cpu/common.c +24 -1
@@ -59,6 +59,7 @@#include <asm/cpu_device_id.h> #include <asm/uv/uv.h> #include <asm/sigframe.h> +#include <asm/traps.h> #include "cpu.h"
@@ -361,7 +362,8 @@ out:/* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = -X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP |X86_CR4_FSGSBASE ; +X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP | +X86_CR4_FSGSBASE |X86_CR4_CET ;static DEFINE_STATIC_KEY_FALSE_RO (cr_pinning);static unsigned long cr4_pinned_bits__ro_after_init ;
@@ -515,6 +517,24 @@ static __init int setup_disable_pku(char *arg)__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +static __always_inline void setup_cet(struct cpuinfo_x86 *c) +{ +u64 msr =CET_ENDBR_EN ; + +if (!HAS_KERNEL_IBT || + !cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +wrmsrl (MSR_IA32_S_CET , msr); +cr4_set_bits (X86_CR4_CET ); + +if (!ibt_selftest ()) { +pr_err ("IBT selftest: Failed!\n" ); +setup_clear_cpu_cap (X86_FEATURE_IBT ); +return ; + } +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
@@ -1632,6 +1652,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)x86_init_rdrand (c);setup_pku (c); +setup_cet (c);/* * Clear/Set all flags overridden by options, need do it
@@ -1698,6 +1719,8 @@ void enable_sep_cpu(void)void __init identify_boot_cpu (void ) {identify_cpu (&boot_cpu_data); +if (HAS_KERNEL_IBT &&cpu_feature_enabled (X86_FEATURE_IBT )) +pr_info ("CET detected: Indirect Branch Tracking enabled\n" );#ifdef CONFIG_X86_32 sysenter_setup ();enable_sep_cpu ();
arch/x86/kernel/idt.c +4 -0
@@ -104,6 +104,10 @@ static const __initconst struct idt_data def_idts[] = { ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),#endif +#ifdef CONFIG_X86_KERNEL_IBT + INTG(X86_TRAP_CP, asm_exc_control_protection), +#endif +#ifdef CONFIG_AMD_MEM_ENCRYPT ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),#endif
arch/x86/kernel/traps.c +74 -0
80 lines not on this page · raw message
Re: [PATCH v4 24/45] x86/ibt: Add IBT feature, MSR and #CP handling
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
arch/x86/kernel/cpu/common.c +4
-0@@ -592,6 +594,24 @@ static __init int setup_disable_pku(char__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +static __always_inline void setup_cet(struct cpuinfo_x86 *c) +{ +u64 msr =CET_ENDBR_EN ; +
arch/x86/kernel/cpu/common.c static __init int setup_disable_pku(char · line 610
Quoted 10 lines
arch/x86/kernel/cpu/common.c +10
-0+if (!HAS_KERNEL_IBT || + !cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +wrmsrl (MSR_IA32_S_CET , msr); +cr4_set_bits (X86_CR4_CET ); + +if (!ibt_selftest ()) { +pr_err ("IBT selftest: Failed!\n" ); +setup_clear_cpu_cap (X86_FEATURE_IBT );
This path wants to clear CR4.CET or MSR_S_CET, or both.
Quoted 30 lines
arch/x86/kernel/cpu/common.c +4
-0+return ; + } +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization arch/x86/kernel/traps.c +17
-0@@ -210,6 +210,80 @@ DEFINE_IDTENTRY(exc_overflow)do_error_trap (regs,0 ,"overflow" ,X86_TRAP_OF ,SIGSEGV ,0 ,NULL ); } +#ifdef CONFIG_X86_KERNEL_IBT + +static __ro_after_init bool ibt_fatal = true; + +void ibt_selftest_ip (void );/* code label defined in asm below */ + +enum cp_error_code { +CP_EC = (1 <<15 ) -1 , + +CP_RET =1 , +CP_IRET =2 , +CP_ENDBR =3 , +CP_RSTRORSSP =4 , +CP_SETSSBSY =5 , + +CP_ENCL =1 <<15 , +};
arch/x86/kernel/traps.c DEFINE_IDTENTRY(exc_overflow) · line 239
Quoted 10 lines
arch/x86/kernel/traps.c +10
-0+ +DEFINE_IDTENTRY_ERRORCODE (exc_control_protection) +{ +if (!cpu_feature_enabled (X86_FEATURE_IBT )) { +pr_err ("Unexpected #CP\n" ); +BUG (); + } + +if (WARN_ON_ONCE (user_mode (regs) || (error_code &CP_EC ) !=CP_ENDBR )) +return ;
Is this wise? #CP is a fault, so this will livelock.
user_mode() wants to turn into an oops. Kernel probably wants to fold into the Unexpected #CP path, but definitely print the error code.
~Andrew
Quoted 52 lines
arch/x86/kernel/traps.c +47
-0+ +if (unlikely (regs->ip == (unsigned long )ibt_selftest_ip)) { + regs->ax =0 ; +return ; + } + +pr_err ("Missing ENDBR: %pS\n" , (void *)instruction_pointer (regs)); +if (!ibt_fatal) { + printk(KERN_DEFAULT CUT_HERE); +__warn (__FILE__, __LINE__, (void *)regs->ip,TAINT_WARN , regs,NULL ); +return ; + } +BUG (); +} + +bool ibt_selftest (void ) +{ +unsigned long ret; + + asm (" lea ibt_selftest_ip(%%rip), %%rax\n\t" +ANNOTATE_RETPOLINE_SAFE +" jmp *%%rax\n\t" +"ibt_selftest_ip:\n\t" +UNWIND_HINT_FUNC +ANNOTATE_NOENDBR +" nop\n\t" + + :"=a" (ret) : :"memory" ); + +return !ret; +} + +static int __init ibt_setup (char *str) +{ +if (!strcmp (str,"off" )) +setup_clear_cpu_cap (X86_FEATURE_IBT ); + +if (!strcmp (str,"warn" )) + ibt_fatal = false; + +return 1 ; +} + +__setup ("ibt=" , ibt_setup); + +#endif /* CONFIG_X86_KERNEL_IBT */ +#ifdef CONFIG_X86_F00F_BUG void handle_invalid_op (struct pt_regs *regs) #else
[tip: x86/core] x86/ibt: Add IBT feature, MSR and #CP handling
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 991625f3dd2cbc4b787deb0213e2bcf8fa264b21 Gitweb: https://git.kernel.org/tip/991625f3dd2cbc4b787deb0213e2bcf8fa264b21 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:35 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:39 +01:00
x86/ibt: Add IBT feature, MSR and #CP handling
The bits required to make the hardware go.. Of note is that, provided the syscall entry points are covered with ENDBR, #CP doesn't need to be an IST because we'll never hit the syscall gap.
--- arch/x86/include/asm/cpu.h | 1 +- arch/x86/include/asm/cpufeatures.h | 1 +- arch/x86/include/asm/idtentry.h | 5 +- arch/x86/include/asm/msr-index.h | 20 ++++- arch/x86/include/asm/traps.h | 2 +- arch/x86/include/uapi/asm/processor-flags.h | 2 +- arch/x86/kernel/cpu/common.c | 25 ++++++- arch/x86/kernel/idt.c | 4 +- arch/x86/kernel/traps.c | 75 ++++++++++++++++++++- 9 files changed, 133 insertions(+), 2 deletions(-)
arch/x86/include/asm/cpu.h +1 -0
@@ -7,6 +7,7 @@#include <linux/topology.h> #include <linux/nodemask.h> #include <linux/percpu.h> +#include <asm/ibt.h> #ifdef CONFIG_SMP
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -390,6 +390,7 @@#define X86_FEATURE_TSXLDTRK (18*32+16)/* TSX Suspend Load Address Tracking */ #define X86_FEATURE_PCONFIG (18*32+18)/* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19)/* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20)/* Indirect Branch Tracking */ #define X86_FEATURE_AVX512_FP16 (18*32+23)/* AVX512 FP16 */ #define X86_FEATURE_SPEC_CTRL (18*32+26)/* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27)/* "" Single Thread Indirect Branch Predictors */
arch/x86/include/asm/idtentry.h +5 -0
@@ -617,6 +617,11 @@ DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);DECLARE_IDTENTRY_RAW_ERRORCODE (X86_TRAP_DF , xenpv_exc_double_fault);#endif +/* #CP */ +#ifdef CONFIG_X86_KERNEL_IBT +DECLARE_IDTENTRY_ERRORCODE (X86_TRAP_CP , exc_control_protection); +#endif +/* #VC */ #ifdef CONFIG_AMD_MEM_ENCRYPT DECLARE_IDTENTRY_VC (X86_TRAP_VC , exc_vmm_communication);
arch/x86/include/asm/msr-index.h +19 -1
@@ -360,11 +360,29 @@#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d -#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x000006a0/* user mode cet */ +#define MSR_IA32_S_CET 0x000006a2/* kernel mode cet */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) + +#define MSR_IA32_PL0_SSP 0x000006a4/* ring-0 shadow stack pointer */ +#define MSR_IA32_PL1_SSP 0x000006a5/* ring-1 shadow stack pointer */ +#define MSR_IA32_PL2_SSP 0x000006a6/* ring-2 shadow stack pointer */ +#define MSR_IA32_PL3_SSP 0x000006a7/* ring-3 shadow stack pointer */ +#define MSR_IA32_INT_SSP_TAB 0x000006a8/* exception shadow stack table */ +/* Hardware P state interface */ #define MSR_PPERF 0x0000064e#define MSR_PERF_LIMIT_REASONS 0x0000064f
arch/x86/include/asm/traps.h +2 -0
@@ -18,6 +18,8 @@ void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);#endif +extern bool ibt_selftest (void ); +#ifdef CONFIG_X86_F00F_BUG /* For handling the FOOF bug */ void handle_invalid_op (struct pt_regs *regs);
arch/x86/include/uapi/asm/processor-flags.h +2 -0
@@ -130,6 +130,8 @@#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)#define X86_CR4_PKE_BIT 22/* enable Protection Keys support */ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) +#define X86_CR4_CET_BIT 23/* enable Control-flow Enforcement Technology */ +#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)/* * x86-64 Task Priority Register, CR8
arch/x86/kernel/cpu/common.c +24 -1
@@ -59,6 +59,7 @@#include <asm/cpu_device_id.h> #include <asm/uv/uv.h> #include <asm/sigframe.h> +#include <asm/traps.h> #include "cpu.h"
@@ -361,7 +362,8 @@ out:/* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = -X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP |X86_CR4_FSGSBASE ; +X86_CR4_SMEP |X86_CR4_SMAP |X86_CR4_UMIP | +X86_CR4_FSGSBASE |X86_CR4_CET ;static DEFINE_STATIC_KEY_FALSE_RO (cr_pinning);static unsigned long cr4_pinned_bits__ro_after_init ;
@@ -515,6 +517,24 @@ static __init int setup_disable_pku(char *arg)__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +static __always_inline void setup_cet(struct cpuinfo_x86 *c) +{ +u64 msr =CET_ENDBR_EN ; + +if (!HAS_KERNEL_IBT || + !cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +wrmsrl (MSR_IA32_S_CET , msr); +cr4_set_bits (X86_CR4_CET ); + +if (!ibt_selftest ()) { +pr_err ("IBT selftest: Failed!\n" ); +setup_clear_cpu_cap (X86_FEATURE_IBT ); +return ; + } +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
@@ -1632,6 +1652,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)x86_init_rdrand (c);setup_pku (c); +setup_cet (c);/* * Clear/Set all flags overridden by options, need do it
@@ -1698,6 +1719,8 @@ void enable_sep_cpu(void)void __init identify_boot_cpu (void ) {identify_cpu (&boot_cpu_data); +if (HAS_KERNEL_IBT &&cpu_feature_enabled (X86_FEATURE_IBT )) +pr_info ("CET detected: Indirect Branch Tracking enabled\n" );#ifdef CONFIG_X86_32 sysenter_setup ();enable_sep_cpu ();
arch/x86/kernel/idt.c +4 -0
@@ -104,6 +104,10 @@ static const __initconst struct idt_data def_idts[] = { ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),#endif +#ifdef CONFIG_X86_KERNEL_IBT + INTG(X86_TRAP_CP, asm_exc_control_protection), +#endif +#ifdef CONFIG_AMD_MEM_ENCRYPT ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),#endif
arch/x86/kernel/traps.c +75 -0
81 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v4 25/45 x86/ibt,kexec: Disable CET on kexec
In reply to Peter Zijlstra--- arch/x86/include/asm/cpu.h | 3 +++ arch/x86/kernel/cpu/common.c | 6 ++++++ arch/x86/kernel/machine_kexec_64.c | 4 +++- arch/x86/kernel/relocate_kernel_64.S | 8 ++++++++ 4 files changed, 20 insertions(+), 1 deletion(-)
arch/x86/include/asm/cpu.h +3 -0
@@ -73,4 +73,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x#else static inline void init_ia32_feat_ctl (struct cpuinfo_x86 *c) {}#endif + +extern __noendbr void cet_disable(void); +#endif /* _ASM_X86_CPU_H */
arch/x86/kernel/cpu/common.c +6 -0
@@ -612,6 +612,12 @@ static __always_inline void setup_cet(st } } +__noendbr void cet_disable(void) +{ +if (cpu_feature_enabled (X86_FEATURE_IBT )) +wrmsrl (MSR_IA32_S_CET ,0 ); +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
arch/x86/kernel/machine_kexec_64.c +3 -1
@@ -27,6 +27,7 @@#include <asm/kexec-bzimage64.h> #include <asm/setup.h> #include <asm/set_memory.h> +#include <asm/cpu.h> #ifdef CONFIG_ACPI /*
@@ -310,6 +311,7 @@ void machine_kexec(struct kimage *image)/* Interrupts aren't acceptable while we reboot */ local_irq_disable ();hw_breakpoint_disable (); +cet_disable ();if (image->preserve_context) {#ifdef CONFIG_X86_IO_APIC
@@ -325,7 +327,7 @@ void machine_kexec(struct kimage *image) } control_page =page_address (image->control_code_page) +PAGE_SIZE ; -memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); +__memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); page_list[PA_CONTROL_PAGE ] =virt_to_phys (control_page); page_list[VA_CONTROL_PAGE ] = (unsigned long )control_page;
arch/x86/kernel/relocate_kernel_64.S +8 -0
@@ -115,6 +115,14 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_ma pushq %rdx /* + * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP + * below. + */ + movq %cr4, %rax + andq $~(X86_CR4_CET), %rax + movq %rax, %cr4 + + /* * Set cr0 to a known state: * - Paging enabled * - Alignment check disabled
[tip: x86/core] x86/ibt,kexec: Disable CET on kexec
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f9a08abb06d0abe51295479118f1f8e4a4325798 Gitweb: https://git.kernel.org/tip/f9a08abb06d0abe51295479118f1f8e4a4325798 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:36 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:33 +01:00
x86/ibt,kexec: Disable CET on kexec
--- arch/x86/include/asm/cpu.h | 3 +++ arch/x86/kernel/cpu/common.c | 6 ++++++ arch/x86/kernel/machine_kexec_64.c | 4 +++- arch/x86/kernel/relocate_kernel_64.S | 8 ++++++++ 4 files changed, 20 insertions(+), 1 deletion(-)
arch/x86/include/asm/cpu.h +3 -0
@@ -73,4 +73,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c);#else static inline void init_ia32_feat_ctl (struct cpuinfo_x86 *c) {}#endif + +extern __noendbr void cet_disable(void); +#endif /* _ASM_X86_CPU_H */
arch/x86/kernel/cpu/common.c +6 -0
@@ -535,6 +535,12 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) } } +__noendbr void cet_disable(void) +{ +if (cpu_feature_enabled (X86_FEATURE_IBT )) +wrmsrl (MSR_IA32_S_CET ,0 ); +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
arch/x86/kernel/machine_kexec_64.c +3 -1
@@ -27,6 +27,7 @@#include <asm/kexec-bzimage64.h> #include <asm/setup.h> #include <asm/set_memory.h> +#include <asm/cpu.h> #ifdef CONFIG_ACPI /*
@@ -310,6 +311,7 @@ void machine_kexec(struct kimage *image)/* Interrupts aren't acceptable while we reboot */ local_irq_disable ();hw_breakpoint_disable (); +cet_disable ();if (image->preserve_context) {#ifdef CONFIG_X86_IO_APIC
@@ -325,7 +327,7 @@ void machine_kexec(struct kimage *image) } control_page =page_address (image->control_code_page) +PAGE_SIZE ; -memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); +__memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); page_list[PA_CONTROL_PAGE ] =virt_to_phys (control_page); page_list[VA_CONTROL_PAGE ] = (unsigned long )control_page;
arch/x86/kernel/relocate_kernel_64.S +8 -0
@@ -115,6 +115,14 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) pushq %rdx /* + * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP + * below. + */ + movq %cr4, %rax + andq $~(X86_CR4_CET), %rax + movq %rax, %cr4 + + /* * Set cr0 to a known state: * - Paging enabled * - Alignment check disabled
[tip: x86/core] x86/ibt,kexec: Disable CET on kexec
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: af22700390c2f1d92dadd3eedf2738525a3a2f3a Gitweb: https://git.kernel.org/tip/af22700390c2f1d92dadd3eedf2738525a3a2f3a Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:36 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:39 +01:00
x86/ibt,kexec: Disable CET on kexec
--- arch/x86/include/asm/cpu.h | 3 +++ arch/x86/kernel/cpu/common.c | 6 ++++++ arch/x86/kernel/machine_kexec_64.c | 4 +++- arch/x86/kernel/relocate_kernel_64.S | 8 ++++++++ 4 files changed, 20 insertions(+), 1 deletion(-)
arch/x86/include/asm/cpu.h +3 -0
@@ -73,4 +73,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c);#else static inline void init_ia32_feat_ctl (struct cpuinfo_x86 *c) {}#endif + +extern __noendbr void cet_disable(void); +#endif /* _ASM_X86_CPU_H */
arch/x86/kernel/cpu/common.c +6 -0
@@ -535,6 +535,12 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) } } +__noendbr void cet_disable(void) +{ +if (cpu_feature_enabled (X86_FEATURE_IBT )) +wrmsrl (MSR_IA32_S_CET ,0 ); +} +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
arch/x86/kernel/machine_kexec_64.c +3 -1
@@ -27,6 +27,7 @@#include <asm/kexec-bzimage64.h> #include <asm/setup.h> #include <asm/set_memory.h> +#include <asm/cpu.h> #ifdef CONFIG_ACPI /*
@@ -310,6 +311,7 @@ void machine_kexec(struct kimage *image)/* Interrupts aren't acceptable while we reboot */ local_irq_disable ();hw_breakpoint_disable (); +cet_disable ();if (image->preserve_context) {#ifdef CONFIG_X86_IO_APIC
@@ -325,7 +327,7 @@ void machine_kexec(struct kimage *image) } control_page =page_address (image->control_code_page) +PAGE_SIZE ; -memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); +__memcpy (control_page, relocate_kernel,KEXEC_CONTROL_CODE_MAX_SIZE ); page_list[PA_CONTROL_PAGE ] =virt_to_phys (control_page); page_list[VA_CONTROL_PAGE ] = (unsigned long )control_page;
arch/x86/kernel/relocate_kernel_64.S +8 -0
@@ -115,6 +115,14 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) pushq %rdx /* + * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP + * below. + */ + movq %cr4, %rax + andq $~(X86_CR4_CET), %rax + movq %rax, %cr4 + + /* * Set cr0 to a known state: * - Paging enabled * - Alignment check disabled
Peter Zijlstra
maintainerPATCH v4 26/45 x86/alternative: Simplify int3_selftest_ip
In reply to Peter ZijlstraSimilar to ibt_selftest_ip, apply the same pattern.
--- arch/x86/kernel/alternative.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
arch/x86/kernel/alternative.c +3 -10
@@ -719,7 +719,7 @@ asm (" .popsection\n" ); -extern __initdata unsigned long int3_selftest_ip;/* defined in asm below */ +extern void int3_selftest_ip (void );/* defined in asm below */ static int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data)
@@ -733,7 +733,7 @@ int3_exception_notify(struct notifier_blif (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);
@@ -757,14 +757,7 @@ static void __init int3_selftest(void)* then trigger the INT3, padded with NOPs to match a CALL instruction * length. */ - asmvolatile ("1: int3; nop; nop; nop; nop\n\t" -".pushsection .init.data,\"aw\"\n\t" - ".align " __ASM_SEL(4, 8) "\n\t" -".type int3_selftest_ip, @object\n\t" - ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t" -"int3_selftest_ip:\n\t" - __ASM_SEL(.long, .quad) " 1b\n\t" - ".popsection\n\t" + asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
[tip: x86/core] x86/alternative: Simplify int3_selftest_ip
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 8cdcaee61f7fe2d35ad8c3b3295ac7bd30244c7f Gitweb: https://git.kernel.org/tip/8cdcaee61f7fe2d35ad8c3b3295ac7bd30244c7f Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:37 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:34 +01:00
x86/alternative: Simplify int3_selftest_ip
Similar to ibt_selftest_ip, apply the same pattern.
--- arch/x86/kernel/alternative.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
arch/x86/kernel/alternative.c +3 -10
@@ -719,7 +719,7 @@ asm (" .popsection\n" ); -extern __initdata unsigned long int3_selftest_ip;/* defined in asm below */ +extern void int3_selftest_ip (void );/* defined in asm below */ static int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data)
@@ -733,7 +733,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *dataif (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);
@@ -757,14 +757,7 @@ static void __init int3_selftest(void)* then trigger the INT3, padded with NOPs to match a CALL instruction * length. */ - asmvolatile ("1: int3; nop; nop; nop; nop\n\t" -".pushsection .init.data,\"aw\"\n\t" - ".align " __ASM_SEL(4, 8) "\n\t" -".type int3_selftest_ip, @object\n\t" - ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t" -"int3_selftest_ip:\n\t" - __ASM_SEL(.long, .quad) " 1b\n\t" - ".popsection\n\t" + asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
[tip: x86/core] x86/alternative: Simplify int3_selftest_ip
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 99c95c5d4f1027130d555fdb27b576520894827d Gitweb: https://git.kernel.org/tip/99c95c5d4f1027130d555fdb27b576520894827d Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:37 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:40 +01:00
x86/alternative: Simplify int3_selftest_ip
Similar to ibt_selftest_ip, apply the same pattern.
--- arch/x86/kernel/alternative.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-)
arch/x86/kernel/alternative.c +7 -14
@@ -719,7 +719,7 @@ asm (" .popsection\n" ); -extern __initdata unsigned long int3_selftest_ip;/* defined in asm below */ +extern void int3_selftest_ip (void );/* defined in asm below */ static int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data)
@@ -733,14 +733,15 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *dataif (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);return NOTIFY_STOP ; } -static void __init int3_selftest (void ) +/* Must be noinline to ensure uniqueness of int3_selftest_ip. */ +static noinline void __init int3_selftest(void) {static __initdata struct notifier_block int3_exception_nb = { .notifier_call = int3_exception_notify,
@@ -753,18 +754,10 @@ static void __init int3_selftest(void)/* * Basically: int3_magic(&val); but really complicated :-) * -* Stick the address of the INT3 instruction into int3_selftest_ip, -* then trigger the INT3, padded with NOPs to match a CALL instruction -* length. +* INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb +* notifier above will emulate CALL for us. */ - asmvolatile ("1: int3; nop; nop; nop; nop\n\t" -".pushsection .init.data,\"aw\"\n\t" - ".align " __ASM_SEL(4, 8) "\n\t" -".type int3_selftest_ip, @object\n\t" - ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t" -"int3_selftest_ip:\n\t" - __ASM_SEL(.long, .quad) " 1b\n\t" - ".popsection\n\t" + asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
Peter Zijlstra
maintainerPATCH v4 27/45 x86/ibt: Disable IBT around firmware
In reply to Peter ZijlstraAssume firmware isn't IBT clean and disable it across calls.
--- arch/x86/include/asm/efi.h | 9 +++++++-- arch/x86/include/asm/ibt.h | 6 ++++++ arch/x86/kernel/apm_32.c | 7 +++++++ arch/x86/kernel/cpu/common.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-)
arch/x86/include/asm/efi.h +7 -2
@@ -7,6 +7,7 @@#include <asm/tlb.h> #include <asm/nospec-branch.h> #include <asm/mmu_context.h> +#include <asm/ibt.h> #include <linux/build_bug.h> #include <linux/kernel.h> #include <linux/pgtable.h>
@@ -120,8 +121,12 @@ extern asmlinkage u64 __efi_call(void *fefi_enter_mm (); \ }) -#define arch_efi_call_virt(p, f, args...) \ - efi_call((void *)p->f, args) \ +#define arch_efi_call_virt(p, f, args...) ({ \ + u64 ret, ibt = ibt_save(); \ + ret = efi_call((void *)p->f, args); \ + ibt_restore(ibt); \ + ret; \ +})#define arch_efi_call_virt_teardown () \ ({ \
arch/x86/include/asm/ibt.h +6 -0
@@ -39,6 +39,9 @@ static inline bool is_endbr(u32 val)return val ==gen_endbr (); } +extern __noendbr u64 ibt_save(void); +extern __noendbr void ibt_restore(u64 save); +#else /* __ASSEMBLY__ */ #ifdef CONFIG_X86_64
@@ -61,6 +64,9 @@ static inline bool is_endbr(u32 val)static inline bool is_endbr (u32 val) {return false; } +static inline u64 ibt_save (void ) {return 0 ; } +static inline void ibt_restore (u64 save) { } +#else /* __ASSEMBLY__ */ #define ENDBR
arch/x86/kernel/apm_32.c +7 -0
@@ -232,6 +232,7 @@#include <asm/paravirt.h> #include <asm/reboot.h> #include <asm/nospec-branch.h> +#include <asm/ibt.h> #if defined(CONFIG_APM_DISPLAY_BLANK ) && defined(CONFIG_VT )extern int (*console_blank_hook)(int );
@@ -598,6 +599,7 @@ static long __apm_bios_call(void *_call)struct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -607,11 +609,13 @@ static long __apm_bios_call(void *_call)apm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ;apm_bios_call_asm (call->func, call->ebx, call->ecx, &call->eax, &call->ebx, &call->ecx, &call->edx, &call->esi);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
@@ -676,6 +680,7 @@ static long __apm_bios_call_simple(voidstruct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -685,10 +690,12 @@ static long __apm_bios_call_simple(voidapm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ; error =apm_bios_call_simple_asm (call->func, call->ebx, call->ecx, &call->eax);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
arch/x86/kernel/cpu/common.c +28 -0
@@ -594,6 +594,34 @@ static __init int setup_disable_pku(char__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +#ifdef CONFIG_X86_KERNEL_IBT + +__noendbr u64 ibt_save(void) +{ +u64 msr =0 ; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); +wrmsrl (MSR_IA32_S_CET , msr & ~CET_ENDBR_EN ); + } + +return msr; +} + +__noendbr void ibt_restore(u64 save) +{ +u64 msr; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); + msr &= ~CET_ENDBR_EN ; + msr |= (save &CET_ENDBR_EN ); +wrmsrl (MSR_IA32_S_CET , msr); + } +} + +#endif + static __always_inline void setup_cet(struct cpuinfo_x86 *c) {u64 msr =CET_ENDBR_EN ;
[tip: x86/core] x86/ibt: Disable IBT around firmware
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 80b6c49877883e5561ec3ecd2a5dd9d0f301eeaf Gitweb: https://git.kernel.org/tip/80b6c49877883e5561ec3ecd2a5dd9d0f301eeaf Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:38 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:34 +01:00
x86/ibt: Disable IBT around firmware
Assume firmware isn't IBT clean and disable it across calls.
--- arch/x86/include/asm/efi.h | 9 +++++++-- arch/x86/include/asm/ibt.h | 6 ++++++ arch/x86/kernel/apm_32.c | 7 +++++++ arch/x86/kernel/cpu/common.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-)
arch/x86/include/asm/efi.h +7 -2
@@ -7,6 +7,7 @@#include <asm/tlb.h> #include <asm/nospec-branch.h> #include <asm/mmu_context.h> +#include <asm/ibt.h> #include <linux/build_bug.h> #include <linux/kernel.h> #include <linux/pgtable.h>
@@ -120,8 +121,12 @@ extern asmlinkage u64 __efi_call(void *fp, ...);efi_enter_mm (); \ }) -#define arch_efi_call_virt(p, f, args...) \ - efi_call((void *)p->f, args) \ +#define arch_efi_call_virt(p, f, args...) ({ \ + u64 ret, ibt = ibt_save(); \ + ret = efi_call((void *)p->f, args); \ + ibt_restore(ibt); \ + ret; \ +})#define arch_efi_call_virt_teardown () \ ({ \
arch/x86/include/asm/ibt.h +6 -0
@@ -52,6 +52,9 @@ static inline bool is_endbr(u32 val)return val ==gen_endbr (); } +extern __noendbr u64 ibt_save(void); +extern __noendbr void ibt_restore(u64 save); +#else /* __ASSEMBLY__ */ #ifdef CONFIG_X86_64
@@ -74,6 +77,9 @@ static inline bool is_endbr(u32 val)static inline bool is_endbr (u32 val) {return false; } +static inline u64 ibt_save (void ) {return 0 ; } +static inline void ibt_restore (u64 save) { } +#else /* __ASSEMBLY__ */ #define ENDBR
arch/x86/kernel/apm_32.c +7 -0
@@ -232,6 +232,7 @@#include <asm/paravirt.h> #include <asm/reboot.h> #include <asm/nospec-branch.h> +#include <asm/ibt.h> #if defined(CONFIG_APM_DISPLAY_BLANK ) && defined(CONFIG_VT )extern int (*console_blank_hook)(int );
@@ -598,6 +599,7 @@ static long __apm_bios_call(void *_call)struct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -607,11 +609,13 @@ static long __apm_bios_call(void *_call)apm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ;apm_bios_call_asm (call->func, call->ebx, call->ecx, &call->eax, &call->ebx, &call->ecx, &call->edx, &call->esi);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
@@ -676,6 +680,7 @@ static long __apm_bios_call_simple(void *_call)struct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -685,10 +690,12 @@ static long __apm_bios_call_simple(void *_call)apm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ; error =apm_bios_call_simple_asm (call->func, call->ebx, call->ecx, &call->eax);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
arch/x86/kernel/cpu/common.c +28 -0
@@ -517,6 +517,34 @@ static __init int setup_disable_pku(char *arg)__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +#ifdef CONFIG_X86_KERNEL_IBT + +__noendbr u64 ibt_save(void) +{ +u64 msr =0 ; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); +wrmsrl (MSR_IA32_S_CET , msr & ~CET_ENDBR_EN ); + } + +return msr; +} + +__noendbr void ibt_restore(u64 save) +{ +u64 msr; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); + msr &= ~CET_ENDBR_EN ; + msr |= (save &CET_ENDBR_EN ); +wrmsrl (MSR_IA32_S_CET , msr); + } +} + +#endif + static __always_inline void setup_cet(struct cpuinfo_x86 *c) {u64 msr =CET_ENDBR_EN ;
[tip: x86/core] x86/ibt: Disable IBT around firmware
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: fe379fa4d199abc52d5b4a256e52cf94eff685cf Gitweb: https://git.kernel.org/tip/fe379fa4d199abc52d5b4a256e52cf94eff685cf Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:38 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:40 +01:00
x86/ibt: Disable IBT around firmware
Assume firmware isn't IBT clean and disable it across calls.
--- arch/x86/include/asm/efi.h | 9 +++++++-- arch/x86/include/asm/ibt.h | 6 ++++++ arch/x86/kernel/apm_32.c | 7 +++++++ arch/x86/kernel/cpu/common.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-)
arch/x86/include/asm/efi.h +7 -2
@@ -7,6 +7,7 @@#include <asm/tlb.h> #include <asm/nospec-branch.h> #include <asm/mmu_context.h> +#include <asm/ibt.h> #include <linux/build_bug.h> #include <linux/kernel.h> #include <linux/pgtable.h>
@@ -120,8 +121,12 @@ extern asmlinkage u64 __efi_call(void *fp, ...);efi_enter_mm (); \ }) -#define arch_efi_call_virt(p, f, args...) \ - efi_call((void *)p->f, args) \ +#define arch_efi_call_virt(p, f, args...) ({ \ + u64 ret, ibt = ibt_save(); \ + ret = efi_call((void *)p->f, args); \ + ibt_restore(ibt); \ + ret; \ +})#define arch_efi_call_virt_teardown () \ ({ \
arch/x86/include/asm/ibt.h +6 -0
@@ -52,6 +52,9 @@ static inline bool is_endbr(u32 val)return val ==gen_endbr (); } +extern __noendbr u64 ibt_save(void); +extern __noendbr void ibt_restore(u64 save); +#else /* __ASSEMBLY__ */ #ifdef CONFIG_X86_64
@@ -74,6 +77,9 @@ static inline bool is_endbr(u32 val)static inline bool is_endbr (u32 val) {return false; } +static inline u64 ibt_save (void ) {return 0 ; } +static inline void ibt_restore (u64 save) { } +#else /* __ASSEMBLY__ */ #define ENDBR
arch/x86/kernel/apm_32.c +7 -0
@@ -232,6 +232,7 @@#include <asm/paravirt.h> #include <asm/reboot.h> #include <asm/nospec-branch.h> +#include <asm/ibt.h> #if defined(CONFIG_APM_DISPLAY_BLANK ) && defined(CONFIG_VT )extern int (*console_blank_hook)(int );
@@ -598,6 +599,7 @@ static long __apm_bios_call(void *_call)struct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -607,11 +609,13 @@ static long __apm_bios_call(void *_call)apm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ;apm_bios_call_asm (call->func, call->ebx, call->ecx, &call->eax, &call->ebx, &call->ecx, &call->edx, &call->esi);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
@@ -676,6 +680,7 @@ static long __apm_bios_call_simple(void *_call)struct desc_struct save_desc_40;struct desc_struct *gdt;struct apm_bios_call *call = _call; +u64 ibt; cpu =get_cpu ();BUG_ON (cpu !=0 );
@@ -685,10 +690,12 @@ static long __apm_bios_call_simple(void *_call)apm_irq_save (flags);firmware_restrict_branch_speculation_start (); + ibt =ibt_save ();APM_DO_SAVE_SEGS ; error =apm_bios_call_simple_asm (call->func, call->ebx, call->ecx, &call->eax);APM_DO_RESTORE_SEGS ; +ibt_restore (ibt);firmware_restrict_branch_speculation_end ();apm_irq_restore (flags); gdt[0x40 /8 ] = save_desc_40;
arch/x86/kernel/cpu/common.c +28 -0
@@ -517,6 +517,34 @@ static __init int setup_disable_pku(char *arg)__setup ("nopku" , setup_disable_pku);#endif /* CONFIG_X86_64 */ +#ifdef CONFIG_X86_KERNEL_IBT + +__noendbr u64 ibt_save(void) +{ +u64 msr =0 ; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); +wrmsrl (MSR_IA32_S_CET , msr & ~CET_ENDBR_EN ); + } + +return msr; +} + +__noendbr void ibt_restore(u64 save) +{ +u64 msr; + +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +rdmsrl (MSR_IA32_S_CET , msr); + msr &= ~CET_ENDBR_EN ; + msr |= (save &CET_ENDBR_EN ); +wrmsrl (MSR_IA32_S_CET , msr); + } +} + +#endif + static __always_inline void setup_cet(struct cpuinfo_x86 *c) {u64 msr =CET_ENDBR_EN ;
Peter Zijlstra
maintainerPATCH v4 28/45 x86/bugs: Disable Retpoline when IBT
In reply to Peter ZijlstraRetpoline and IBT are mutually exclusive. IBT relies on indirect branches (JMP/CALL *%reg) while retpoline avoids them by design.
Demote to LFENCE on IBT enabled hardware.
--- arch/x86/kernel/cpu/bugs.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
arch/x86/kernel/cpu/bugs.c +13 -0
@@ -891,6 +891,7 @@ static void __init spectre_v2_select_mit {enum spectre_v2_mitigation_cmd cmd =spectre_v2_parse_cmdline ();enum spectre_v2_mitigation mode =SPECTRE_V2_NONE ; +bool silent_demote = false;/* * If the CPU is not affected and the command line mode is NONE or AUTO
@@ -906,6 +907,7 @@ static void __init spectre_v2_select_mitcase SPECTRE_V2_CMD_FORCE :case SPECTRE_V2_CMD_AUTO : + silent_demote = true;if (boot_cpu_has (X86_FEATURE_IBRS_ENHANCED )) { mode =SPECTRE_V2_IBRS_ENHANCED ;/* Force it so VMEXIT will restore correctly */
@@ -938,6 +940,7 @@ static void __init spectre_v2_select_mit retpoline_amd:if (!boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) {pr_err ("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n" ); + silent_demote = false; goto retpoline_generic; } mode =SPECTRE_V2_RETPOLINE_AMD ;
@@ -947,6 +950,16 @@ static void __init spectre_v2_select_mit retpoline_generic: mode =SPECTRE_V2_RETPOLINE_GENERIC ;setup_force_cpu_cap (X86_FEATURE_RETPOLINE ); + +/* +* ROP defeats IBT, make sure not to use Retpolines and IBT together. +*/ +if (HAS_KERNEL_IBT &&cpu_feature_enabled (X86_FEATURE_IBT )) { +if (!silent_demote) +pr_warn ("Spectre mitigation: Switching to LFENCE due to IBT\n" ); + mode =SPECTRE_V2_RETPOLINE_AMD ; +setup_force_cpu_cap (X86_FEATURE_RETPOLINE_AMD ); + } } specv2_set_mode:
Peter Zijlstra
maintainerPATCH v4 29/45 x86/ibt: Annotate text references
In reply to Peter ZijlstraAnnotate away some of the generic code references. This is things where we take the address of a symbol for exception handling or return addresses (eg. context switch).
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/alternative.c | 10 ++++++++-- arch/x86/kernel/head_64.S | 4 ++++ arch/x86/kernel/kprobes/core.c | 1 + arch/x86/kernel/relocate_kernel_64.S | 2 ++ arch/x86/lib/error-inject.c | 2 ++ arch/x86/lib/retpoline.S | 1 + 8 files changed, 25 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +6 -0
@@ -277,6 +277,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" SYM_CODE_START(ret_from_fork) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // copy_thread movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */
@@ -569,6 +570,7 @@ SYM_CODE_END(\asmsym) .align 16 .globl __irqentry_text_end __irqentry_text_end: + ANNOTATE_NOENDBR SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
@@ -650,6 +652,7 @@ SYM_INNER_LABEL(early_xen_iret_patch, SY #endif SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // exc_double_fault /* * This may fault. Non-paranoid faults on return to userspace are * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
@@ -744,6 +747,7 @@ SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs .Lgs_change: + ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs
@@ -1322,6 +1326,7 @@ SYM_CODE_START(asm_exc_nmi) #endif repeat_nmi: + ANNOTATE_NOENDBR // this code /* * If there was a nested NMI, the first NMI's iret will return * here. But NMIs are still enabled and we can take another
@@ -1350,6 +1355,7 @@ SYM_CODE_START(asm_exc_nmi) .endr subq $(5*8), %rsp end_repeat_nmi: + ANNOTATE_NOENDBR // this code /* * Everything below this point can be preempted by a nested NMI.
arch/x86/entry/entry_64_compat.S +1 -0
@@ -148,6 +148,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_af popfq jmp .Lsysenter_flags_fixed SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // is_sysenter_singlestep SYM_CODE_END(entry_SYSENTER_compat) /*
arch/x86/kernel/alternative.c +8 -2
@@ -713,6 +713,7 @@ asm (" .pushsection .init.text, \"ax\", @progbits\n" " .type int3_magic, @function\n" "int3_magic:\n" +ANNOTATE_NOENDBR " movl $1, (%" _ASM_ARG1")\n" ASM_RET " .size int3_magic, .-int3_magic\n"
@@ -724,16 +725,19 @@ extern void int3_selftest_ip(void); /* dstatic int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data) { +unsigned long selftest = (unsigned long )&int3_selftest_ip;struct die_args *args = data;struct pt_regs *regs = args->regs; +OPTIMIZER_HIDE_VAR (selftest); +if (!regs ||user_mode (regs))return NOTIFY_DONE ;if (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != selftest)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);
@@ -757,7 +761,9 @@ static void __init int3_selftest(void)* then trigger the INT3, padded with NOPs to match a CALL instruction * length. */ - asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" + asmvolatile ("int3_selftest_ip:\n\t" +ANNOTATE_NOENDBR +" int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
arch/x86/kernel/head_64.S +4 -0
@@ -99,6 +99,7 @@ SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded a mapped page table.
@@ -127,6 +128,7 @@ SYM_CODE_START(secondary_startup_64) */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * Retrieve the modifier (SME encryption mask if SME is active) to be
@@ -192,6 +194,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_ jmp *%rax 1: UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // above /* * We must switch to a new descriptor in kernel space for the GDT
@@ -299,6 +302,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_ pushq %rax # target address in negative space lretq .Lafter_lret: + ANNOTATE_NOENDBR SYM_CODE_END(secondary_startup_64) #include "verify_cpu.S"
arch/x86/kernel/kprobes/core.c +1 -0
@@ -1033,6 +1033,7 @@ asm(".type __kretprobe_trampoline, @function\n" "__kretprobe_trampoline:\n" #ifdef CONFIG_X86_64 +ANNOTATE_NOENDBR /* Push a fake return address to tell the unwinder it's a kretprobe. */ " pushq $__kretprobe_trampoline\n" UNWIND_HINT_FUNC
arch/x86/kernel/relocate_kernel_64.S +2 -0
@@ -42,6 +42,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_kernel) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * %rdi indirection_page * %rsi page_list
@@ -223,6 +224,7 @@ SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax movq %rax, %cr4
arch/x86/lib/error-inject.c +2 -0
@@ -3,6 +3,7 @@#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h> +#include <linux/objtool.h> asmlinkage void just_return_func(void);
@@ -11,6 +12,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" +ANNOTATE_NOENDBR ASM_RET ".size just_return_func, .-just_return_func\n" );
arch/x86/lib/retpoline.S +1 -0
@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\re .align RETPOLINE_THUNK_SIZE SYM_CODE_START(__x86_indirect_thunk_array) + ANNOTATE_NOENDBR // apply_retpolines #define GEN(reg) THUNK reg #include <asm/GEN-for-each-reg.h>
[tip: x86/core] x86/ibt: Annotate text references
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: eabab5ecc8993fd6f71ce6bba0da6b22381b946e Gitweb: https://git.kernel.org/tip/eabab5ecc8993fd6f71ce6bba0da6b22381b946e Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:40 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:34 +01:00
x86/ibt: Annotate text references
Annotate away some of the generic code references. This is things where we take the address of a symbol for exception handling or return addresses (eg. context switch).
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/alternative.c | 10 ++++++++-- arch/x86/kernel/head_64.S | 4 ++++ arch/x86/kernel/kprobes/core.c | 1 + arch/x86/kernel/relocate_kernel_64.S | 2 ++ arch/x86/lib/error-inject.c | 2 ++ arch/x86/lib/retpoline.S | 1 + 8 files changed, 25 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +6 -0
@@ -277,6 +277,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" SYM_CODE_START(ret_from_fork) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // copy_thread movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */
@@ -569,6 +570,7 @@ __irqentry_text_start: .align 16 .globl __irqentry_text_end __irqentry_text_end: + ANNOTATE_NOENDBR SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
@@ -650,6 +652,7 @@ SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL) #endif SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // exc_double_fault /* * This may fault. Non-paranoid faults on return to userspace are * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
@@ -744,6 +747,7 @@ SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs .Lgs_change: + ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs
@@ -1322,6 +1326,7 @@ first_nmi: #endif repeat_nmi: + ANNOTATE_NOENDBR // this code /* * If there was a nested NMI, the first NMI's iret will return * here. But NMIs are still enabled and we can take another
@@ -1350,6 +1355,7 @@ repeat_nmi: .endr subq $(5*8), %rsp end_repeat_nmi: + ANNOTATE_NOENDBR // this code /* * Everything below this point can be preempted by a nested NMI.
arch/x86/entry/entry_64_compat.S +1 -0
@@ -148,6 +148,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) popfq jmp .Lsysenter_flags_fixed SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // is_sysenter_singlestep SYM_CODE_END(entry_SYSENTER_compat) /*
arch/x86/kernel/alternative.c +8 -2
@@ -713,6 +713,7 @@ asm (" .pushsection .init.text, \"ax\", @progbits\n" " .type int3_magic, @function\n" "int3_magic:\n" +ANNOTATE_NOENDBR " movl $1, (%" _ASM_ARG1")\n" ASM_RET " .size int3_magic, .-int3_magic\n"
@@ -724,16 +725,19 @@ extern void int3_selftest_ip(void); /* defined in asm below */static int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data) { +unsigned long selftest = (unsigned long )&int3_selftest_ip;struct die_args *args = data;struct pt_regs *regs = args->regs; +OPTIMIZER_HIDE_VAR (selftest); +if (!regs ||user_mode (regs))return NOTIFY_DONE ;if (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != selftest)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);
@@ -757,7 +761,9 @@ static void __init int3_selftest(void)* then trigger the INT3, padded with NOPs to match a CALL instruction * length. */ - asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" + asmvolatile ("int3_selftest_ip:\n\t" +ANNOTATE_NOENDBR +" int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
arch/x86/kernel/head_64.S +4 -0
@@ -99,6 +99,7 @@ SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded a mapped page table.
@@ -127,6 +128,7 @@ SYM_CODE_START(secondary_startup_64) */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * Retrieve the modifier (SME encryption mask if SME is active) to be
@@ -192,6 +194,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) jmp *%rax 1: UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // above /* * We must switch to a new descriptor in kernel space for the GDT
@@ -299,6 +302,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) pushq %rax # target address in negative space lretq .Lafter_lret: + ANNOTATE_NOENDBR SYM_CODE_END(secondary_startup_64) #include "verify_cpu.S"
arch/x86/kernel/kprobes/core.c +1 -0
@@ -1033,6 +1033,7 @@ asm(".type __kretprobe_trampoline, @function\n" "__kretprobe_trampoline:\n" #ifdef CONFIG_X86_64 +ANNOTATE_NOENDBR /* Push a fake return address to tell the unwinder it's a kretprobe. */ " pushq $__kretprobe_trampoline\n" UNWIND_HINT_FUNC
arch/x86/kernel/relocate_kernel_64.S +2 -0
@@ -42,6 +42,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_kernel) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * %rdi indirection_page * %rsi page_list
@@ -223,6 +224,7 @@ SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax movq %rax, %cr4
arch/x86/lib/error-inject.c +2 -0
@@ -3,6 +3,7 @@#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h> +#include <linux/objtool.h> asmlinkage void just_return_func(void);
@@ -11,6 +12,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" +ANNOTATE_NOENDBR ASM_RET ".size just_return_func, .-just_return_func\n" );
arch/x86/lib/retpoline.S +1 -0
@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) .align RETPOLINE_THUNK_SIZE SYM_CODE_START(__x86_indirect_thunk_array) + ANNOTATE_NOENDBR // apply_retpolines #define GEN(reg) THUNK reg #include <asm/GEN-for-each-reg.h>
[tip: x86/core] x86/ibt: Annotate text references
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 3e3f069504344c241f89737e4af014f83fca0b27 Gitweb: https://git.kernel.org/tip/3e3f069504344c241f89737e4af014f83fca0b27 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:40 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:40 +01:00
x86/ibt: Annotate text references
Annotate away some of the generic code references. This is things where we take the address of a symbol for exception handling or return addresses (eg. context switch).
--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/alternative.c | 10 ++++++++-- arch/x86/kernel/head_64.S | 4 ++++ arch/x86/kernel/kprobes/core.c | 1 + arch/x86/kernel/relocate_kernel_64.S | 2 ++ arch/x86/lib/error-inject.c | 2 ++ arch/x86/lib/retpoline.S | 1 + 8 files changed, 25 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +6 -0
@@ -277,6 +277,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" SYM_CODE_START(ret_from_fork) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // copy_thread movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */
@@ -569,6 +570,7 @@ __irqentry_text_start: .align 16 .globl __irqentry_text_end __irqentry_text_end: + ANNOTATE_NOENDBR SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
@@ -650,6 +652,7 @@ SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL) #endif SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // exc_double_fault /* * This may fault. Non-paranoid faults on return to userspace are * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
@@ -744,6 +747,7 @@ SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs .Lgs_change: + ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs
@@ -1322,6 +1326,7 @@ first_nmi: #endif repeat_nmi: + ANNOTATE_NOENDBR // this code /* * If there was a nested NMI, the first NMI's iret will return * here. But NMIs are still enabled and we can take another
@@ -1350,6 +1355,7 @@ repeat_nmi: .endr subq $(5*8), %rsp end_repeat_nmi: + ANNOTATE_NOENDBR // this code /* * Everything below this point can be preempted by a nested NMI.
arch/x86/entry/entry_64_compat.S +1 -0
@@ -148,6 +148,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) popfq jmp .Lsysenter_flags_fixed SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // is_sysenter_singlestep SYM_CODE_END(entry_SYSENTER_compat) /*
arch/x86/kernel/alternative.c +8 -2
@@ -713,6 +713,7 @@ asm (" .pushsection .init.text, \"ax\", @progbits\n" " .type int3_magic, @function\n" "int3_magic:\n" +ANNOTATE_NOENDBR " movl $1, (%" _ASM_ARG1")\n" ASM_RET " .size int3_magic, .-int3_magic\n"
@@ -724,16 +725,19 @@ extern void int3_selftest_ip(void); /* defined in asm below */static int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data) { +unsigned long selftest = (unsigned long )&int3_selftest_ip;struct die_args *args = data;struct pt_regs *regs = args->regs; +OPTIMIZER_HIDE_VAR (selftest); +if (!regs ||user_mode (regs))return NOTIFY_DONE ;if (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != selftest)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);
@@ -757,7 +761,9 @@ static noinline void __init int3_selftest(void)* INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb * notifier above will emulate CALL for us. */ - asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" + asmvolatile ("int3_selftest_ip:\n\t" +ANNOTATE_NOENDBR +" int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");
arch/x86/kernel/head_64.S +4 -0
@@ -99,6 +99,7 @@ SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded a mapped page table.
@@ -127,6 +128,7 @@ SYM_CODE_START(secondary_startup_64) */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * Retrieve the modifier (SME encryption mask if SME is active) to be
@@ -192,6 +194,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) jmp *%rax 1: UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // above /* * We must switch to a new descriptor in kernel space for the GDT
@@ -299,6 +302,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) pushq %rax # target address in negative space lretq .Lafter_lret: + ANNOTATE_NOENDBR SYM_CODE_END(secondary_startup_64) #include "verify_cpu.S"
arch/x86/kernel/kprobes/core.c +1 -0
@@ -1033,6 +1033,7 @@ asm(".type __kretprobe_trampoline, @function\n" "__kretprobe_trampoline:\n" #ifdef CONFIG_X86_64 +ANNOTATE_NOENDBR /* Push a fake return address to tell the unwinder it's a kretprobe. */ " pushq $__kretprobe_trampoline\n" UNWIND_HINT_FUNC
arch/x86/kernel/relocate_kernel_64.S +2 -0
@@ -42,6 +42,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_kernel) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * %rdi indirection_page * %rsi page_list
@@ -223,6 +224,7 @@ SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax movq %rax, %cr4
arch/x86/lib/error-inject.c +2 -0
@@ -3,6 +3,7 @@#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h> +#include <linux/objtool.h> asmlinkage void just_return_func(void);
@@ -11,6 +12,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" +ANNOTATE_NOENDBR ASM_RET ".size just_return_func, .-just_return_func\n" );
arch/x86/lib/retpoline.S +1 -0
@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) .align RETPOLINE_THUNK_SIZE SYM_CODE_START(__x86_indirect_thunk_array) + ANNOTATE_NOENDBR // apply_retpolines #define GEN(reg) THUNK reg #include <asm/GEN-for-each-reg.h>
Masami Hiramatsu
maintainerRe: [PATCH v4 29/45] x86/ibt: Annotate text references
In reply to Peter ZijlstraQuoted 3 lines · On Tue, 08 Mar 2022 16:30:40 +0100 Peter Zijlstra <[email protected]> wrote:
Annotate away some of the generic code references. This is things where we take the address of a symbol for exception handling or return addresses (eg. context switch).
Ah, I got it. I need this annotate lines for rethook too.
Thank you,
Quoted 207 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/entry/entry_64.S | 6 ++++++ arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/alternative.c | 10 ++++++++-- arch/x86/kernel/head_64.S | 4 ++++ arch/x86/kernel/kprobes/core.c | 1 + arch/x86/kernel/relocate_kernel_64.S | 2 ++ arch/x86/lib/error-inject.c | 2 ++ arch/x86/lib/retpoline.S | 1 + 8 files changed, 25 insertions(+), 2 deletions(-)arch/x86/entry/entry_64.S +6
-0@@ -277,6 +277,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" SYM_CODE_START(ret_from_fork) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // copy_thread movq %rax, %rdi call schedule_tail /* rdi: 'prev' task parameter */@@ -569,6 +570,7 @@ SYM_CODE_END(\asmsym) .align 16 .globl __irqentry_text_end __irqentry_text_end: + ANNOTATE_NOENDBR SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)@@ -650,6 +652,7 @@ SYM_INNER_LABEL(early_xen_iret_patch, SY #endif SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // exc_double_fault /* * This may fault. Non-paranoid faults on return to userspace are * handled by fixup_bad_iret. These include #SS, #GP, and #NP.@@ -744,6 +747,7 @@ SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs .Lgs_change: + ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs@@ -1322,6 +1326,7 @@ SYM_CODE_START(asm_exc_nmi) #endif repeat_nmi: + ANNOTATE_NOENDBR // this code /* * If there was a nested NMI, the first NMI's iret will return * here. But NMIs are still enabled and we can take another@@ -1350,6 +1355,7 @@ SYM_CODE_START(asm_exc_nmi) .endr subq $(5*8), %rsp end_repeat_nmi: + ANNOTATE_NOENDBR // this code /* * Everything below this point can be preempted by a nested NMI.arch/x86/entry/entry_64_compat.S +1
-0@@ -148,6 +148,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_af popfq jmp .Lsysenter_flags_fixed SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) + ANNOTATE_NOENDBR // is_sysenter_singlestep SYM_CODE_END(entry_SYSENTER_compat) /*arch/x86/kernel/alternative.c +8
-2@@ -713,6 +713,7 @@ asm (" .pushsection .init.text, \"ax\", @progbits\n" " .type int3_magic, @function\n" "int3_magic:\n" +ANNOTATE_NOENDBR " movl $1, (%" _ASM_ARG1")\n" ASM_RET " .size int3_magic, .-int3_magic\n" @@ -724,16 +725,19 @@ extern void int3_selftest_ip(void); /* dstatic int __init int3_exception_notify (struct notifier_block *self,unsigned long val,void *data) { +unsigned long selftest = (unsigned long )&int3_selftest_ip;struct die_args *args = data;struct pt_regs *regs = args->regs; +OPTIMIZER_HIDE_VAR (selftest); +if (!regs ||user_mode (regs))return NOTIFY_DONE ;if (val !=DIE_INT3 )return NOTIFY_DONE ; -if (regs->ip -INT3_INSN_SIZE != (unsigned long )&int3_selftest_ip) +if (regs->ip -INT3_INSN_SIZE != selftest)return NOTIFY_DONE ;int3_emulate_call (regs, (unsigned long )&int3_magic);@@ -757,7 +761,9 @@ static void __init int3_selftest(void)* then trigger the INT3, padded with NOPs to match a CALL instruction * length. */ - asmvolatile ("int3_selftest_ip: int3; nop; nop; nop; nop\n\t" + asmvolatile ("int3_selftest_ip:\n\t" +ANNOTATE_NOENDBR +" int3; nop; nop; nop; nop\n\t" : ASM_CALL_CONSTRAINT : __ASM_SEL_RAW(a, D) (&val) : "memory");arch/x86/kernel/head_64.S +4
-0@@ -99,6 +99,7 @@ SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded a mapped page table.@@ -127,6 +128,7 @@ SYM_CODE_START(secondary_startup_64) */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * Retrieve the modifier (SME encryption mask if SME is active) to be@@ -192,6 +194,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_ jmp *%rax 1: UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // above /* * We must switch to a new descriptor in kernel space for the GDT@@ -299,6 +302,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_ pushq %rax # target address in negative space lretq .Lafter_lret: + ANNOTATE_NOENDBR SYM_CODE_END(secondary_startup_64) #include "verify_cpu.S"arch/x86/kernel/kprobes/core.c +1
-0@@ -1033,6 +1033,7 @@ asm(".type __kretprobe_trampoline, @function\n" "__kretprobe_trampoline:\n" #ifdef CONFIG_X86_64 +ANNOTATE_NOENDBR /* Push a fake return address to tell the unwinder it's a kretprobe. */ " pushq $__kretprobe_trampoline\n" UNWIND_HINT_FUNC arch/x86/kernel/relocate_kernel_64.S +2
-0@@ -42,6 +42,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_kernel) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR /* * %rdi indirection_page * %rsi page_list@@ -223,6 +224,7 @@ SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) UNWIND_HINT_EMPTY + ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax movq %rax, %cr4arch/x86/lib/error-inject.c +2
-0@@ -3,6 +3,7 @@#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h> +#include <linux/objtool.h> asmlinkage void just_return_func(void);@@ -11,6 +12,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" +ANNOTATE_NOENDBR ASM_RET ".size just_return_func, .-just_return_func\n" );arch/x86/lib/retpoline.S +1
-0@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\re .align RETPOLINE_THUNK_SIZE SYM_CODE_START(__x86_indirect_thunk_array) + ANNOTATE_NOENDBR // apply_retpolines #define GEN(reg) THUNK reg #include <asm/GEN-for-each-reg.h>
Signature · 1 line
Masami Hiramatsu <[email protected]>
Re: [PATCH v4 29/45] x86/ibt: Annotate text references
Quoted 9 lines · Masami Hiramatsu
Quoted 3 lines · On Tue, 08 Mar 2022 16:30:40 +0100 Peter Zijlstra <[email protected]> wrote:
Annotate away some of the generic code references. This is things where we take the address of a symbol for exception handling or return addresses (eg. context switch).
Ah, I got it. I need this annotate lines for rethook too.
Masami,
Peter already proposed a fix:
https://lore.kernel.org/lkml/[email protected]/
Peter Zijlstra
maintainerPATCH v4 30/45 x86/ibt,ftrace: Annotate ftrace code patching
In reply to Peter ZijlstraThese are code patching sites, not indirect targets.
--- arch/x86/kernel/ftrace_64.S | 7 +++++++ 1 file changed, 7 insertions(+)
arch/x86/kernel/ftrace_64.S +7 -0
@@ -145,6 +145,7 @@ SYM_FUNC_START(ftrace_caller) movq %rcx, RSP(%rsp) SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -155,6 +156,7 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SY movq $0, CS(%rsp) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Handlers can change the RIP */
@@ -169,6 +171,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBA * layout here. */ SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue SYM_FUNC_END(ftrace_caller);
@@ -192,6 +195,7 @@ SYM_FUNC_START(ftrace_regs_caller) /* save_mcount_regs fills in first two parameters */ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -221,6 +225,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_pt leaq (%rsp), %rcx SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Copy flags back to SS, to restore them */
@@ -248,6 +253,7 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_ */ testq %rax, %rax SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jnz 1f restore_mcount_regs
@@ -261,6 +267,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, * to the return. */ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue /* Swap the flags with orig_rax */
[tip: x86/core] x86/ibt,ftrace: Annotate ftrace code patching
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 84683d8f7ebaf8409e4400842116bd378a017057 Gitweb: https://git.kernel.org/tip/84683d8f7ebaf8409e4400842116bd378a017057 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:41 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:35 +01:00
x86/ibt,ftrace: Annotate ftrace code patching
These are code patching sites, not indirect targets.
--- arch/x86/kernel/ftrace_64.S | 7 +++++++ 1 file changed, 7 insertions(+)
arch/x86/kernel/ftrace_64.S +7 -0
@@ -145,6 +145,7 @@ SYM_FUNC_START(ftrace_caller) movq %rcx, RSP(%rsp) SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -155,6 +156,7 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) movq $0, CS(%rsp) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Handlers can change the RIP */
@@ -169,6 +171,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) * layout here. */ SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue SYM_FUNC_END(ftrace_caller);
@@ -192,6 +195,7 @@ SYM_FUNC_START(ftrace_regs_caller) /* save_mcount_regs fills in first two parameters */ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -221,6 +225,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) leaq (%rsp), %rcx SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Copy flags back to SS, to restore them */
@@ -248,6 +253,7 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) */ testq %rax, %rax SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jnz 1f restore_mcount_regs
@@ -261,6 +267,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) * to the return. */ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue /* Swap the flags with orig_rax */
[tip: x86/core] x86/ibt,ftrace: Annotate ftrace code patching
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 3215de84c06d747bb748b98945add83e3ec8a6e2 Gitweb: https://git.kernel.org/tip/3215de84c06d747bb748b98945add83e3ec8a6e2 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:41 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:41 +01:00
x86/ibt,ftrace: Annotate ftrace code patching
These are code patching sites, not indirect targets.
--- arch/x86/kernel/ftrace_64.S | 7 +++++++ 1 file changed, 7 insertions(+)
arch/x86/kernel/ftrace_64.S +7 -0
@@ -145,6 +145,7 @@ SYM_FUNC_START(ftrace_caller) movq %rcx, RSP(%rsp) SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -155,6 +156,7 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) movq $0, CS(%rsp) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Handlers can change the RIP */
@@ -169,6 +171,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) * layout here. */ SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue SYM_FUNC_END(ftrace_caller);
@@ -192,6 +195,7 @@ SYM_FUNC_START(ftrace_regs_caller) /* save_mcount_regs fills in first two parameters */ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ movq function_trace_op(%rip), %rdx
@@ -221,6 +225,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) leaq (%rsp), %rcx SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) + ANNOTATE_NOENDBR call ftrace_stub /* Copy flags back to SS, to restore them */
@@ -248,6 +253,7 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) */ testq %rax, %rax SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jnz 1f restore_mcount_regs
@@ -261,6 +267,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) * to the return. */ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) + ANNOTATE_NOENDBR jmp ftrace_epilogue /* Swap the flags with orig_rax */
Peter Zijlstra
maintainerPATCH v4 31/45 x86/ibt,sev: Annotations
In reply to Peter ZijlstraNo IBT on AMD so far.. probably correct, who knows.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/head_64.S | 2 ++ 3 files changed, 4 insertions(+)
arch/x86/entry/entry_64.S +1 -0
@@ -95,6 +95,7 @@ SYM_CODE_START(entry_SYSCALL_64) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */
arch/x86/entry/entry_64_compat.S +1 -0
@@ -214,6 +214,7 @@ SYM_CODE_START(entry_SYSCALL_compat) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER32_DS /* pt_regs->ss */
arch/x86/kernel/head_64.S +2 -0
@@ -332,6 +332,7 @@ SYM_CODE_END(start_cpu0) */ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
@@ -439,6 +440,7 @@ SYM_CODE_END(early_idt_handler_common) */ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
[tip: x86/core] x86/ibt,sev: Annotations
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: e58b0dfcd8634a758ec0fc14b200c20c70013635 Gitweb: https://git.kernel.org/tip/e58b0dfcd8634a758ec0fc14b200c20c70013635 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:42 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:35 +01:00
x86/ibt,sev: Annotations
No IBT on AMD so far.. probably correct, who knows.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/head_64.S | 2 ++ 3 files changed, 4 insertions(+)
arch/x86/entry/entry_64.S +1 -0
@@ -95,6 +95,7 @@ SYM_CODE_START(entry_SYSCALL_64) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */
arch/x86/entry/entry_64_compat.S +1 -0
@@ -214,6 +214,7 @@ SYM_CODE_START(entry_SYSCALL_compat) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER32_DS /* pt_regs->ss */
arch/x86/kernel/head_64.S +2 -0
@@ -332,6 +332,7 @@ SYM_CODE_END(start_cpu0) */ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
@@ -439,6 +440,7 @@ SYM_CODE_END(early_idt_handler_common) */ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
[tip: x86/core] x86/ibt,sev: Annotations
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: e8d61bdf0fdfaeaf35fb5a63d6e67e60038b88e0 Gitweb: https://git.kernel.org/tip/e8d61bdf0fdfaeaf35fb5a63d6e67e60038b88e0 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:42 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:41 +01:00
x86/ibt,sev: Annotations
No IBT on AMD so far.. probably correct, who knows.
--- arch/x86/entry/entry_64.S | 1 + arch/x86/entry/entry_64_compat.S | 1 + arch/x86/kernel/head_64.S | 2 ++ 3 files changed, 4 insertions(+)
arch/x86/entry/entry_64.S +1 -0
@@ -95,6 +95,7 @@ SYM_CODE_START(entry_SYSCALL_64) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */
arch/x86/entry/entry_64_compat.S +1 -0
@@ -214,6 +214,7 @@ SYM_CODE_START(entry_SYSCALL_compat) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL) + ANNOTATE_NOENDBR /* Construct struct pt_regs on stack */ pushq $__USER32_DS /* pt_regs->ss */
arch/x86/kernel/head_64.S +2 -0
@@ -332,6 +332,7 @@ SYM_CODE_END(start_cpu0) */ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
@@ -439,6 +440,7 @@ SYM_CODE_END(early_idt_handler_common) */ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 + ENDBR /* Build pt_regs */ PUSH_AND_CLEAR_REGS
Peter Zijlstra
maintainerPATCH v4 32/45 x86/ibt: Dont generate ENDBR in .discard.text
In reply to Peter ZijlstraHaving ENDBR in discarded sections can easily lead to relocations into discarded sections which the linkers aren't really fond of. Objtool also shouldn't generate them, but why tempt fate.
--- arch/x86/include/asm/setup.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/setup.h +2 -1
@@ -8,6 +8,7 @@#include <linux/linkage.h> #include <asm/page_types.h> +#include <asm/ibt.h> #ifdef __i386__
@@ -119,7 +120,7 @@ void *extend_brk(size_t size, size_t ali* executable.) */ #define RESERVE_BRK (name,sz) \ - static void __section(".discard.text") __used notrace \ + static void __section(".discard.text") __noendbr __used notrace \ __brk_reservation_fn_##name##__(void) { \ asm volatile ( \ ".pushsection .brk_reservation,\"aw\",@nobits;" \
[tip: x86/core] x86/ibt: Dont generate ENDBR in .discard.text
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c0b5a22ebc8f40ef5df8c0ab9797c2b62d2566dd Gitweb: https://git.kernel.org/tip/c0b5a22ebc8f40ef5df8c0ab9797c2b62d2566dd Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:43 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:35 +01:00
x86/ibt: Dont generate ENDBR in .discard.text
Having ENDBR in discarded sections can easily lead to relocations into discarded sections which the linkers aren't really fond of. Objtool also shouldn't generate them, but why tempt fate.
--- arch/x86/include/asm/setup.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/setup.h +2 -1
@@ -8,6 +8,7 @@#include <linux/linkage.h> #include <asm/page_types.h> +#include <asm/ibt.h> #ifdef __i386__
@@ -119,7 +120,7 @@ void *extend_brk(size_t size, size_t align);* executable.) */ #define RESERVE_BRK (name,sz) \ - static void __section(".discard.text") __used notrace \ + static void __section(".discard.text") __noendbr __used notrace \ __brk_reservation_fn_##name##__(void) { \ asm volatile ( \ ".pushsection .brk_reservation,\"aw\",@nobits;" \
[tip: x86/core] x86/ibt: Dont generate ENDBR in .discard.text
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2b6ff7dea670a4623fae1d2349806fc7f8e305d1 Gitweb: https://git.kernel.org/tip/2b6ff7dea670a4623fae1d2349806fc7f8e305d1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:43 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:42 +01:00
x86/ibt: Dont generate ENDBR in .discard.text
Having ENDBR in discarded sections can easily lead to relocations into discarded sections which the linkers aren't really fond of. Objtool also shouldn't generate them, but why tempt fate.
--- arch/x86/include/asm/setup.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/setup.h +2 -1
@@ -8,6 +8,7 @@#include <linux/linkage.h> #include <asm/page_types.h> +#include <asm/ibt.h> #ifdef __i386__
@@ -119,7 +120,7 @@ void *extend_brk(size_t size, size_t align);* executable.) */ #define RESERVE_BRK (name,sz) \ - static void __section(".discard.text") __used notrace \ + static void __section(".discard.text") __noendbr __used notrace \ __brk_reservation_fn_##name##__(void) { \ asm volatile ( \ ".pushsection .brk_reservation,\"aw\",@nobits;" \
Peter Zijlstra
maintainerPATCH v4 33/45 x86/ibt: Ensure module init/exit points have references
In reply to Peter ZijlstraSince the references to the module init/exit points only have external references, a module LTO run will consider them 'unused' and seal them, leading to an immediate fail on module load.
--- include/linux/cfi.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
include/linux/cfi.h +10 -1
@@ -34,8 +34,17 @@ static inline void cfi_module_remove(str#else /* !CONFIG_CFI_CLANG */ -#define __CFI_ADDRESSABLE (fn, __attr) +#ifdef CONFIG_X86_KERNEL_IBT + +#define __CFI_ADDRESSABLE (fn, __attr) \ + const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn + +#endif /* CONFIG_X86_KERNEL_IBT */ #endif /* CONFIG_CFI_CLANG */ +#ifndef __CFI_ADDRESSABLE +#define __CFI_ADDRESSABLE (fn, __attr) +#endif +#endif /* _LINUX_CFI_H */
[tip: x86/core] x86/ibt: Ensure module init/exit points have references
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 43d8f5801147a9d3998261d491e2d9a3de6e05eb Gitweb: https://git.kernel.org/tip/43d8f5801147a9d3998261d491e2d9a3de6e05eb Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:44 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:35 +01:00
x86/ibt: Ensure module init/exit points have references
Since the references to the module init/exit points only have external references, a module LTO run will consider them 'unused' and seal them, leading to an immediate fail on module load.
--- include/linux/cfi.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
include/linux/cfi.h +10 -1
@@ -34,8 +34,17 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr#else /* !CONFIG_CFI_CLANG */ -#define __CFI_ADDRESSABLE (fn, __attr) +#ifdef CONFIG_X86_KERNEL_IBT + +#define __CFI_ADDRESSABLE (fn, __attr) \ + const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn + +#endif /* CONFIG_X86_KERNEL_IBT */ #endif /* CONFIG_CFI_CLANG */ +#ifndef __CFI_ADDRESSABLE +#define __CFI_ADDRESSABLE (fn, __attr) +#endif +#endif /* _LINUX_CFI_H */
[tip: x86/core] x86/ibt: Ensure module init/exit points have references
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: cb9010f87dcbcdbb51cc96b922c6260848cecbd1 Gitweb: https://git.kernel.org/tip/cb9010f87dcbcdbb51cc96b922c6260848cecbd1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:44 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:42 +01:00
x86/ibt: Ensure module init/exit points have references
Since the references to the module init/exit points only have external references, a module LTO run will consider them 'unused' and seal them, leading to an immediate fail on module load.
--- include/linux/cfi.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
include/linux/cfi.h +10 -1
@@ -34,8 +34,17 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr#else /* !CONFIG_CFI_CLANG */ -#define __CFI_ADDRESSABLE (fn, __attr) +#ifdef CONFIG_X86_KERNEL_IBT + +#define __CFI_ADDRESSABLE (fn, __attr) \ + const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn + +#endif /* CONFIG_X86_KERNEL_IBT */ #endif /* CONFIG_CFI_CLANG */ +#ifndef __CFI_ADDRESSABLE +#define __CFI_ADDRESSABLE (fn, __attr) +#endif +#endif /* _LINUX_CFI_H */
Peter Zijlstra
maintainerPATCH v4 34/45 objtool: Rename --duplicate to --lto
In reply to Peter ZijlstraIn order to prepare for LTO like objtool runs for modules, rename the duplicate argument to lto.
--- scripts/link-vmlinux.sh | 2 +- tools/objtool/builtin-check.c | 4 ++-- tools/objtool/check.c | 7 ++++++- tools/objtool/include/objtool/builtin.h | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-)
scripts/link-vmlinux.sh +1 -1
@@ -115,7 +115,7 @@ objtool_link() objtoolcmd="orc generate" fi - objtoolopt="${objtoolopt} --duplicate" + objtoolopt="${objtoolopt} --lto" if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then objtoolopt="${objtoolopt} --mcount"
tools/objtool/builtin-check.c +2 -2
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -40,7 +40,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('b' ,"backtrace" , &backtrace,"unwind on error" ),OPT_BOOLEAN ('a' ,"uaccess" , &uaccess,"enable uaccess checking" ),OPT_BOOLEAN ('s' ,"stats" , &stats,"print statistics" ), -OPT_BOOLEAN ('d' ,"duplicate" , &validate_dup,"duplicate validation for vmlinux.o" ), +OPT_BOOLEAN (0 ,"lto" , <o,"whole-archive like runs" ),OPT_BOOLEAN ('n' ,"noinstr" , &noinstr,"noinstr validation for vmlinux.o" ),OPT_BOOLEAN ('l' ,"vmlinux" , &vmlinux,"vmlinux.o validation" ), OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
tools/objtool/check.c +6 -1
@@ -3501,6 +3501,11 @@ int check(struct objtool_file *file) {int ret, warnings =0 ; +if (lto && !(vmlinux || module)) { +fprintf (stderr,"--lto requires: --vmlinux or --module\n" ); +return 1 ; + } +arch_initial_func_cfi_state (&initial_func_cfi);init_cfi_state (&init_cfi);init_cfi_state (&func_cfi);
@@ -3521,7 +3526,7 @@ int check(struct objtool_file *file)if (list_empty (&file->insn_list)) goto out; -if (vmlinux && !validate_dup) { +if (vmlinux && !lto) { ret =validate_vmlinux_functions (file);if (ret <0 ) goto out;
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
[tip: x86/core] objtool: Rename --duplicate to --lto
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 5af720d0563adc652ac1e98bc4ab423ba1336927 Gitweb: https://git.kernel.org/tip/5af720d0563adc652ac1e98bc4ab423ba1336927 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:45 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:36 +01:00
objtool: Rename --duplicate to --lto
In order to prepare for LTO like objtool runs for modules, rename the duplicate argument to lto.
--- scripts/link-vmlinux.sh | 2 +- tools/objtool/builtin-check.c | 4 ++-- tools/objtool/check.c | 7 ++++++- tools/objtool/include/objtool/builtin.h | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-)
scripts/link-vmlinux.sh +1 -1
@@ -115,7 +115,7 @@ objtool_link() objtoolcmd="orc generate" fi - objtoolopt="${objtoolopt} --duplicate" + objtoolopt="${objtoolopt} --lto" if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then objtoolopt="${objtoolopt} --mcount"
tools/objtool/builtin-check.c +2 -2
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -40,7 +40,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('b' ,"backtrace" , &backtrace,"unwind on error" ),OPT_BOOLEAN ('a' ,"uaccess" , &uaccess,"enable uaccess checking" ),OPT_BOOLEAN ('s' ,"stats" , &stats,"print statistics" ), -OPT_BOOLEAN ('d' ,"duplicate" , &validate_dup,"duplicate validation for vmlinux.o" ), +OPT_BOOLEAN (0 ,"lto" , <o,"whole-archive like runs" ),OPT_BOOLEAN ('n' ,"noinstr" , &noinstr,"noinstr validation for vmlinux.o" ),OPT_BOOLEAN ('l' ,"vmlinux" , &vmlinux,"vmlinux.o validation" ), OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
tools/objtool/check.c +6 -1
@@ -3499,6 +3499,11 @@ int check(struct objtool_file *file) {int ret, warnings =0 ; +if (lto && !(vmlinux || module)) { +fprintf (stderr,"--lto requires: --vmlinux or --module\n" ); +return 1 ; + } +arch_initial_func_cfi_state (&initial_func_cfi);init_cfi_state (&init_cfi);init_cfi_state (&func_cfi);
@@ -3519,7 +3524,7 @@ int check(struct objtool_file *file)if (list_empty (&file->insn_list)) goto out; -if (vmlinux && !validate_dup) { +if (vmlinux && !lto) { ret =validate_vmlinux_functions (file);if (ret <0 ) goto out;
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
[tip: x86/core] objtool: Rename --duplicate to --lto
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 53f7109ef957315ab53205ba3a3f4f48874c0428 Gitweb: https://git.kernel.org/tip/53f7109ef957315ab53205ba3a3f4f48874c0428 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:45 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:42 +01:00
objtool: Rename --duplicate to --lto
In order to prepare for LTO like objtool runs for modules, rename the duplicate argument to lto.
--- scripts/link-vmlinux.sh | 2 +- tools/objtool/builtin-check.c | 4 ++-- tools/objtool/check.c | 7 ++++++- tools/objtool/include/objtool/builtin.h | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-)
scripts/link-vmlinux.sh +1 -1
@@ -115,7 +115,7 @@ objtool_link() objtoolcmd="orc generate" fi - objtoolopt="${objtoolopt} --duplicate" + objtoolopt="${objtoolopt} --lto" if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then objtoolopt="${objtoolopt} --mcount"
tools/objtool/builtin-check.c +2 -2
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -40,7 +40,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('b' ,"backtrace" , &backtrace,"unwind on error" ),OPT_BOOLEAN ('a' ,"uaccess" , &uaccess,"enable uaccess checking" ),OPT_BOOLEAN ('s' ,"stats" , &stats,"print statistics" ), -OPT_BOOLEAN ('d' ,"duplicate" , &validate_dup,"duplicate validation for vmlinux.o" ), +OPT_BOOLEAN (0 ,"lto" , <o,"whole-archive like runs" ),OPT_BOOLEAN ('n' ,"noinstr" , &noinstr,"noinstr validation for vmlinux.o" ),OPT_BOOLEAN ('l' ,"vmlinux" , &vmlinux,"vmlinux.o validation" ), OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
tools/objtool/check.c +6 -1
@@ -3499,6 +3499,11 @@ int check(struct objtool_file *file) {int ret, warnings =0 ; +if (lto && !(vmlinux || module)) { +fprintf (stderr,"--lto requires: --vmlinux or --module\n" ); +return 1 ; + } +arch_initial_func_cfi_state (&initial_func_cfi);init_cfi_state (&init_cfi);init_cfi_state (&func_cfi);
@@ -3519,7 +3524,7 @@ int check(struct objtool_file *file)if (list_empty (&file->insn_list)) goto out; -if (vmlinux && !validate_dup) { +if (vmlinux && !lto) { ret =validate_vmlinux_functions (file);if (ret <0 ) goto out;
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup, sls, dryrun; + lto, vmlinux, mcount, noinstr, backup, sls, dryrun;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
Peter Zijlstra
maintainerPATCH v4 35/45 objtool: Ignore extra-symbol code
In reply to Peter ZijlstraThere's a fun implementation detail on linking STB_WEAK symbols. When the linker combines two translation units, where one contains a weak function and the other an override for it. It simply strips the STB_WEAK symbol from the symbol table, but doesn't actually remove the code.
The result is that when objtool is ran in a whole-archive kind of way, it will encounter *heaps* of unused (and unreferenced) code. All rudiments of weak functions.
Additionally, when a weak implementation is split into a .cold subfunction that .cold symbol is left in place, even though completely unused.
Teach objtool to ignore such rudiments by searching for symbol holes; that is, code ranges that fall outside the given symbol bounds. Specifically, ignore a sequence of unreachable instruction iff they occupy a single hole, additionally ignore any .cold subfunctions referenced.
Both ld.bfd and ld.lld behave like this. LTO builds otoh can (and do) properly DCE weak functions.
--- tools/objtool/check.c | 43 +++++++++++++++++++++++++ tools/objtool/elf.c | 60 ++++++++++++++++++++++++++++++++++++ tools/objtool/include/objtool/elf.h | 1 3 files changed, 104 insertions(+)
tools/objtool/check.c +43 -0
@@ -3577,6 +3577,49 @@ static bool ignore_unreachable_insn(stru !strcmp(insn->sec->name, ".altinstr_aux"))return true; +/* +* Whole archive runs might encounder dead code from weak symbols. +* This is where the linker will have dropped the weak symbol in +* favour of a regular symbol, but leaves the code in place. +* +* In this case we'll find a piece of code (whole function) that is not +* covered by a !section symbol. Ignore them. +*/ +if (!insn->func && lto) { +int size =find_symbol_hole_containing (insn->sec, insn->offset); +unsigned long end = insn->offset + size; + +if (!size)/* not a hole */ +return false; + +if (size <0 )/* hole until the end */ +return true; + +sec_for_each_insn_continue (file, insn) { +/* +* If we reach a visited instruction at or before the +* end of the hole, ignore the unreachable. +*/ +if (insn->visited) +return true; + +if (insn->offset >= end) +break ; + +/* +* If this hole jumps to a .cold function, mark it ignore too. +*/ +if (insn->jump_dest && insn->jump_dest->func && +strstr (insn->jump_dest->func->name,".cold" )) { +struct instruction *dest = insn->jump_dest; +func_for_each_insn (file, dest->func, dest) + dest->ignore = true; + } + } + +return false; + } +if (!insn->func)return false;
tools/objtool/elf.c +60 -0
72 lines not on this page · raw message
tools/objtool/include/objtool/elf.h +1 -0
@@ -152,6 +152,7 @@ struct symbol *find_func_by_offset(strucstruct symbol *find_symbol_by_offset (struct section *sec,unsigned long offset);struct symbol *find_symbol_by_name (const struct elf *elf,const char *name);struct symbol *find_symbol_containing (const struct section *sec,unsigned long offset); +int find_symbol_hole_containing (const struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest (const struct elf *elf,struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest_range (const struct elf *elf,struct section *sec,unsigned long offset,unsigned int len);
[tip: x86/core] objtool: Ignore extra-symbol code
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 7e37550e601931b79c4cd70ae13c67a01e66d981 Gitweb: https://git.kernel.org/tip/7e37550e601931b79c4cd70ae13c67a01e66d981 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:46 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:36 +01:00
objtool: Ignore extra-symbol code
There's a fun implementation detail on linking STB_WEAK symbols. When the linker combines two translation units, where one contains a weak function and the other an override for it. It simply strips the STB_WEAK symbol from the symbol table, but doesn't actually remove the code.
The result is that when objtool is ran in a whole-archive kind of way, it will encounter *heaps* of unused (and unreferenced) code. All rudiments of weak functions.
Additionally, when a weak implementation is split into a .cold subfunction that .cold symbol is left in place, even though completely unused.
Teach objtool to ignore such rudiments by searching for symbol holes; that is, code ranges that fall outside the given symbol bounds. Specifically, ignore a sequence of unreachable instruction iff they occupy a single hole, additionally ignore any .cold subfunctions referenced.
Both ld.bfd and ld.lld behave like this. LTO builds otoh can (and do) properly DCE weak functions.
--- tools/objtool/check.c | 43 ++++++++++++++++++++- tools/objtool/elf.c | 60 ++++++++++++++++++++++++++++- tools/objtool/include/objtool/elf.h | 1 +- 3 files changed, 104 insertions(+)
tools/objtool/check.c +43 -0
@@ -3346,6 +3346,49 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio !strcmp(insn->sec->name, ".altinstr_aux"))return true; +/* +* Whole archive runs might encounder dead code from weak symbols. +* This is where the linker will have dropped the weak symbol in +* favour of a regular symbol, but leaves the code in place. +* +* In this case we'll find a piece of code (whole function) that is not +* covered by a !section symbol. Ignore them. +*/ +if (!insn->func && lto) { +int size =find_symbol_hole_containing (insn->sec, insn->offset); +unsigned long end = insn->offset + size; + +if (!size)/* not a hole */ +return false; + +if (size <0 )/* hole until the end */ +return true; + +sec_for_each_insn_continue (file, insn) { +/* +* If we reach a visited instruction at or before the +* end of the hole, ignore the unreachable. +*/ +if (insn->visited) +return true; + +if (insn->offset >= end) +break ; + +/* +* If this hole jumps to a .cold function, mark it ignore too. +*/ +if (insn->jump_dest && insn->jump_dest->func && +strstr (insn->jump_dest->func->name,".cold" )) { +struct instruction *dest = insn->jump_dest; +func_for_each_insn (file, dest->func, dest) + dest->ignore = true; + } + } + +return false; + } +if (!insn->func)return false;
tools/objtool/elf.c +60 -0
72 lines not on this page · raw message
tools/objtool/include/objtool/elf.h +1 -0
@@ -152,6 +152,7 @@ struct symbol *find_func_by_offset(struct section *sec, unsigned long offset);struct symbol *find_symbol_by_offset (struct section *sec,unsigned long offset);struct symbol *find_symbol_by_name (const struct elf *elf,const char *name);struct symbol *find_symbol_containing (const struct section *sec,unsigned long offset); +int find_symbol_hole_containing (const struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest (const struct elf *elf,struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest_range (const struct elf *elf,struct section *sec,unsigned long offset,unsigned int len);
[tip: x86/core] objtool: Ignore extra-symbol code
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 4adb23686795e9c88e3217b5d7b4524c0da9d04f Gitweb: https://git.kernel.org/tip/4adb23686795e9c88e3217b5d7b4524c0da9d04f Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:46 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:43 +01:00
objtool: Ignore extra-symbol code
There's a fun implementation detail on linking STB_WEAK symbols. When the linker combines two translation units, where one contains a weak function and the other an override for it. It simply strips the STB_WEAK symbol from the symbol table, but doesn't actually remove the code.
The result is that when objtool is ran in a whole-archive kind of way, it will encounter *heaps* of unused (and unreferenced) code. All rudiments of weak functions.
Additionally, when a weak implementation is split into a .cold subfunction that .cold symbol is left in place, even though completely unused.
Teach objtool to ignore such rudiments by searching for symbol holes; that is, code ranges that fall outside the given symbol bounds. Specifically, ignore a sequence of unreachable instruction iff they occupy a single hole, additionally ignore any .cold subfunctions referenced.
Both ld.bfd and ld.lld behave like this. LTO builds otoh can (and do) properly DCE weak functions.
--- tools/objtool/check.c | 43 ++++++++++++++++++++- tools/objtool/elf.c | 60 ++++++++++++++++++++++++++++- tools/objtool/include/objtool/elf.h | 1 +- 3 files changed, 104 insertions(+)
tools/objtool/check.c +43 -0
@@ -3346,6 +3346,49 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio !strcmp(insn->sec->name, ".altinstr_aux"))return true; +/* +* Whole archive runs might encounder dead code from weak symbols. +* This is where the linker will have dropped the weak symbol in +* favour of a regular symbol, but leaves the code in place. +* +* In this case we'll find a piece of code (whole function) that is not +* covered by a !section symbol. Ignore them. +*/ +if (!insn->func && lto) { +int size =find_symbol_hole_containing (insn->sec, insn->offset); +unsigned long end = insn->offset + size; + +if (!size)/* not a hole */ +return false; + +if (size <0 )/* hole until the end */ +return true; + +sec_for_each_insn_continue (file, insn) { +/* +* If we reach a visited instruction at or before the +* end of the hole, ignore the unreachable. +*/ +if (insn->visited) +return true; + +if (insn->offset >= end) +break ; + +/* +* If this hole jumps to a .cold function, mark it ignore too. +*/ +if (insn->jump_dest && insn->jump_dest->func && +strstr (insn->jump_dest->func->name,".cold" )) { +struct instruction *dest = insn->jump_dest; +func_for_each_insn (file, dest->func, dest) + dest->ignore = true; + } + } + +return false; + } +if (!insn->func)return false;
tools/objtool/elf.c +60 -0
72 lines not on this page · raw message
tools/objtool/include/objtool/elf.h +1 -0
@@ -152,6 +152,7 @@ struct symbol *find_func_by_offset(struct section *sec, unsigned long offset);struct symbol *find_symbol_by_offset (struct section *sec,unsigned long offset);struct symbol *find_symbol_by_name (const struct elf *elf,const char *name);struct symbol *find_symbol_containing (const struct section *sec,unsigned long offset); +int find_symbol_hole_containing (const struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest (const struct elf *elf,struct section *sec,unsigned long offset);struct reloc *find_reloc_by_dest_range (const struct elf *elf,struct section *sec,unsigned long offset,unsigned int len);
Peter Zijlstra
maintainerPATCH v4 36/45 x86: Mark stop_this_cpu() __noreturn
In reply to Peter Zijlstravmlinux.o: warning: objtool: smp_stop_nmi_callback()+0x2b: unreachable instruction
0000 0000000000047cf0 <smp_stop_nmi_callback>:
...
0026 47d16: e8 00 00 00 00 call 47d1b <smp_stop_nmi_callback+0x2b> 47d17: R_X86_64_PLT32 stop_this_cpu-0x4 002b 47d1b: b8 01 00 00 00 mov $0x1,%eax
--- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/process.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/include/asm/processor.h +1 -1
@@ -837,7 +837,7 @@ bool xen_set_default_idle(void);#define xen_set_default_idle 0#endif -void stop_this_cpu (void *dummy); +void __noreturn stop_this_cpu (void *dummy);void microcode_check (void );enum l1tf_mitigations {
arch/x86/kernel/process.c +1 -1
@@ -747,7 +747,7 @@ bool xen_set_default_idle(void) }#endif -void stop_this_cpu (void *dummy) +void __noreturn stop_this_cpu (void *dummy) {local_irq_disable ();/*
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct o"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , + "stop_this_cpu", };if (!func)
[tip: x86/core] x86: Mark stop_this_cpu() __noreturn
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: b109f2f6ab58de3cd4db3adeb81bbe56fb8793ea Gitweb: https://git.kernel.org/tip/b109f2f6ab58de3cd4db3adeb81bbe56fb8793ea Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:47 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:36 +01:00
x86: Mark stop_this_cpu() __noreturn
vmlinux.o: warning: objtool: smp_stop_nmi_callback()+0x2b: unreachable instruction
0000 0000000000047cf0 <smp_stop_nmi_callback>:
...
0026 47d16: e8 00 00 00 00 call 47d1b <smp_stop_nmi_callback+0x2b> 47d17: R_X86_64_PLT32 stop_this_cpu-0x4 002b 47d1b: b8 01 00 00 00 mov $0x1,%eax
--- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/process.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/include/asm/processor.h +1 -1
@@ -835,7 +835,7 @@ bool xen_set_default_idle(void);#define xen_set_default_idle 0#endif -void stop_this_cpu (void *dummy); +void __noreturn stop_this_cpu (void *dummy);void microcode_check (void );enum l1tf_mitigations {
arch/x86/kernel/process.c +1 -1
@@ -747,7 +747,7 @@ bool xen_set_default_idle(void) }#endif -void stop_this_cpu (void *dummy) +void __noreturn stop_this_cpu (void *dummy) {local_irq_disable ();/*
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , + "stop_this_cpu", };if (!func)
[tip: x86/core] x86: Mark stop_this_cpu() __noreturn
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f9cdf7ca57cada055f61ef6d0eb4db21c3f200db Gitweb: https://git.kernel.org/tip/f9cdf7ca57cada055f61ef6d0eb4db21c3f200db Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:47 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:43 +01:00
x86: Mark stop_this_cpu() __noreturn
vmlinux.o: warning: objtool: smp_stop_nmi_callback()+0x2b: unreachable instruction
0000 0000000000047cf0 <smp_stop_nmi_callback>:
...
0026 47d16: e8 00 00 00 00 call 47d1b <smp_stop_nmi_callback+0x2b> 47d17: R_X86_64_PLT32 stop_this_cpu-0x4 002b 47d1b: b8 01 00 00 00 mov $0x1,%eax
--- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/process.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/include/asm/processor.h +1 -1
@@ -835,7 +835,7 @@ bool xen_set_default_idle(void);#define xen_set_default_idle 0#endif -void stop_this_cpu (void *dummy); +void __noreturn stop_this_cpu (void *dummy);void microcode_check (void );enum l1tf_mitigations {
arch/x86/kernel/process.c +1 -1
@@ -747,7 +747,7 @@ bool xen_set_default_idle(void) }#endif -void stop_this_cpu (void *dummy) +void __noreturn stop_this_cpu (void *dummy) {local_irq_disable ();/*
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , + "stop_this_cpu", };if (!func)
Peter Zijlstra
maintainerPATCH v4 37/45 exit: Mark do_group_exit() __noreturn
In reply to Peter Zijlstravmlinux.o: warning: objtool: get_signal()+0x108: unreachable instruction
0000 000000000007f930 <get_signal>:
...
0103 7fa33: e8 00 00 00 00 call 7fa38 <get_signal+0x108> 7fa34: R_X86_64_PLT32 do_group_exit-0x4 0108 7fa38: 41 8b 45 74 mov 0x74(%r13),%eax
--- include/linux/sched/task.h | 2 +- kernel/exit.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
include/linux/sched/task.h +1 -1
@@ -79,7 +79,7 @@ static inline void exit_thread(struct ta { }#endif -extern void do_group_exit (int ); +extern __noreturn void do_group_exit (int );extern void exit_files (struct task_struct *);extern void exit_itimers (struct signal_struct *);
kernel/exit.c +1 -1
@@ -907,7 +907,7 @@ SYSCALL_DEFINE1(exit, int, error_code)* Take down every thread in the group. This is called by fatal signals * as well as by sys_exit_group (below). */ -void +void __noreturn do_group_exit (int exit_code) {struct signal_struct *sig = current->signal;
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct o"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , +"do_group_exit" , "stop_this_cpu", };
[tip: x86/core] exit: Mark do_group_exit() __noreturn
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: fb604370c4b1c1c17fa9c3f0d60bbe8c9494edd9 Gitweb: https://git.kernel.org/tip/fb604370c4b1c1c17fa9c3f0d60bbe8c9494edd9 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:48 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:37 +01:00
exit: Mark do_group_exit() __noreturn
vmlinux.o: warning: objtool: get_signal()+0x108: unreachable instruction
0000 000000000007f930 <get_signal>:
...
0103 7fa33: e8 00 00 00 00 call 7fa38 <get_signal+0x108> 7fa34: R_X86_64_PLT32 do_group_exit-0x4 0108 7fa38: 41 8b 45 74 mov 0x74(%r13),%eax
--- include/linux/sched/task.h | 2 +- kernel/exit.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
include/linux/sched/task.h +1 -1
@@ -79,7 +79,7 @@ static inline void exit_thread(struct task_struct *tsk) { }#endif -extern void do_group_exit (int ); +extern __noreturn void do_group_exit (int );extern void exit_files (struct task_struct *);extern void exit_itimers (struct signal_struct *);
kernel/exit.c +1 -1
@@ -906,7 +906,7 @@ SYSCALL_DEFINE1(exit, int, error_code)* Take down every thread in the group. This is called by fatal signals * as well as by sys_exit_group (below). */ -void +void __noreturn do_group_exit (int exit_code) {struct signal_struct *sig = current->signal;
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , +"do_group_exit" , "stop_this_cpu", };
[tip: x86/core] exit: Mark do_group_exit() __noreturn
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: eae654f1c21216daa9fbb92591c0d9f5ae46cfc5 Gitweb: https://git.kernel.org/tip/eae654f1c21216daa9fbb92591c0d9f5ae46cfc5 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:48 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:43 +01:00
exit: Mark do_group_exit() __noreturn
vmlinux.o: warning: objtool: get_signal()+0x108: unreachable instruction
0000 000000000007f930 <get_signal>:
...
0103 7fa33: e8 00 00 00 00 call 7fa38 <get_signal+0x108> 7fa34: R_X86_64_PLT32 do_group_exit-0x4 0108 7fa38: 41 8b 45 74 mov 0x74(%r13),%eax
--- include/linux/sched/task.h | 2 +- kernel/exit.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
include/linux/sched/task.h +1 -1
@@ -79,7 +79,7 @@ static inline void exit_thread(struct task_struct *tsk) { }#endif -extern void do_group_exit (int ); +extern __noreturn void do_group_exit (int );extern void exit_files (struct task_struct *);extern void exit_itimers (struct signal_struct *);
kernel/exit.c +1 -1
@@ -906,7 +906,7 @@ SYSCALL_DEFINE1(exit, int, error_code)* Take down every thread in the group. This is called by fatal signals * as well as by sys_exit_group (below). */ -void +void __noreturn do_group_exit (int exit_code) {struct signal_struct *sig = current->signal;
tools/objtool/check.c +1 -0
@@ -181,6 +181,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,"kunit_try_catch_throw" ,"xen_start_kernel" ,"cpu_bringup_and_idle" , +"do_group_exit" , "stop_this_cpu", };
Peter Zijlstra
maintainerPATCH v4 38/45 objtool: Rework ASM_REACHABLE
In reply to Peter ZijlstraCurrently ASM_REACHABLE only works for UD2 instructions; reorder things to also allow over-riding dead_end_function().
To that end:
- Mark INSN_BUG instructions in decode_instructions(), this saves having to iterate all instructions yet again.
- Have add_call_destinations() set insn->dead_end for dead_end_function() calls.
- Move add_dead_ends() *after* add_call_destinations() such that ASM_REACHABLE can clear the ->dead_end mark.
- have validate_branch() only check ->dead_end.
--- tools/objtool/check.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-)
tools/objtool/check.c +20 -13
@@ -394,6 +394,14 @@ static int decode_instructions(struct obif (ret) goto err; +/* +* By default, "ud2" is a dead end unless otherwise +* annotated, because GCC 7 inserts it for certain +* divide-by-zero cases. +*/ +if (insn->type ==INSN_BUG ) + insn->dead_end = true; +hash_add (file->insn_hash, &insn->hash,sec_offset_hash (sec, insn->offset));list_add_tail (&insn->list, &file->insn_list); nr_insns++;
@@ -532,14 +541,6 @@ static int add_dead_ends(struct objtool_struct instruction *insn;/* -* By default, "ud2" is a dead end unless otherwise annotated, because -* GCC 7 inserts it for certain divide-by-zero cases. -*/ -for_each_insn (file, insn) -if (insn->type ==INSN_BUG ) - insn->dead_end = true; - -/* * Check for manually annotated dead ends. */ sec =find_section_by_name (file->elf,".rela.discard.unreachable" );
@@ -1174,6 +1175,9 @@ static void annotate_call_site(struct oblist_add_tail (&insn->call_node, &file->mcount_loc_list);return ; } + +if (!sibling &&dead_end_function (file, sym)) + insn->dead_end = true; } static void add_call_dest(struct objtool_file *file, struct instruction *insn,
@@ -2198,10 +2202,6 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =add_dead_ends (file); -if (ret) -return ret; -add_ignores (file);add_uaccess_safe (file);
@@ -2247,6 +2247,14 @@ static int decode_sections(struct objtooif (ret)return ret; +/* +* Must be after add_call_destinations() such that it can override +* dead_end_function() marks. +*/ + ret =add_dead_ends (file); +if (ret) +return ret; + ret =add_jump_table_alts (file);if (ret)return ret;
@@ -3363,7 +3371,7 @@ static int validate_branch(struct objtooreturn 1 ; } -if (dead_end_function (file, insn->call_dest)) +if (insn->dead_end)return 0 ;break ;
[tip: x86/core] objtool: Rework ASM_REACHABLE
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 0d5b64134b49881d62029a1a9923c604507e973b Gitweb: https://git.kernel.org/tip/0d5b64134b49881d62029a1a9923c604507e973b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:49 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:37 +01:00
objtool: Rework ASM_REACHABLE
Currently ASM_REACHABLE only works for UD2 instructions; reorder things to also allow over-riding dead_end_function().
To that end:
- Mark INSN_BUG instructions in decode_instructions(), this saves having to iterate all instructions yet again.
- Have add_call_destinations() set insn->dead_end for dead_end_function() calls.
- Move add_dead_ends() *after* add_call_destinations() such that ASM_REACHABLE can clear the ->dead_end mark.
- have validate_branch() only check ->dead_end.
--- tools/objtool/check.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)
tools/objtool/check.c +20 -13
@@ -394,6 +394,14 @@ static int decode_instructions(struct objtool_file *file)if (ret) goto err; +/* +* By default, "ud2" is a dead end unless otherwise +* annotated, because GCC 7 inserts it for certain +* divide-by-zero cases. +*/ +if (insn->type ==INSN_BUG ) + insn->dead_end = true; +hash_add (file->insn_hash, &insn->hash,sec_offset_hash (sec, insn->offset));list_add_tail (&insn->list, &file->insn_list); nr_insns++;
@@ -523,14 +531,6 @@ static int add_dead_ends(struct objtool_file *file)struct instruction *insn;/* -* By default, "ud2" is a dead end unless otherwise annotated, because -* GCC 7 inserts it for certain divide-by-zero cases. -*/ -for_each_insn (file, insn) -if (insn->type ==INSN_BUG ) - insn->dead_end = true; - -/* * Check for manually annotated dead ends. */ sec =find_section_by_name (file->elf,".rela.discard.unreachable" );
@@ -1113,6 +1113,9 @@ static void annotate_call_site(struct objtool_file *file,list_add_tail (&insn->call_node, &file->mcount_loc_list);return ; } + +if (!sibling &&dead_end_function (file, sym)) + insn->dead_end = true; } static void add_call_dest(struct objtool_file *file, struct instruction *insn,
@@ -2088,10 +2091,6 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; - ret =add_dead_ends (file); -if (ret) -return ret; -add_ignores (file);add_uaccess_safe (file);
@@ -2130,6 +2129,14 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; +/* +* Must be after add_call_destinations() such that it can override +* dead_end_function() marks. +*/ + ret =add_dead_ends (file); +if (ret) +return ret; + ret =add_jump_table_alts (file);if (ret)return ret;
@@ -3137,7 +3144,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,return 1 ; } -if (dead_end_function (file, insn->call_dest)) +if (insn->dead_end)return 0 ;break ;
[tip: x86/core] objtool: Rework ASM_REACHABLE
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 0e5b613b4d4be3345dda349fb90dd73d2103302f Gitweb: https://git.kernel.org/tip/0e5b613b4d4be3345dda349fb90dd73d2103302f Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:49 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:44 +01:00
objtool: Rework ASM_REACHABLE
Currently ASM_REACHABLE only works for UD2 instructions; reorder things to also allow over-riding dead_end_function().
To that end:
- Mark INSN_BUG instructions in decode_instructions(), this saves having to iterate all instructions yet again.
- Have add_call_destinations() set insn->dead_end for dead_end_function() calls.
- Move add_dead_ends() *after* add_call_destinations() such that ASM_REACHABLE can clear the ->dead_end mark.
- have validate_branch() only check ->dead_end.
--- tools/objtool/check.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)
tools/objtool/check.c +20 -13
@@ -395,6 +395,14 @@ static int decode_instructions(struct objtool_file *file)if (ret) goto err; +/* +* By default, "ud2" is a dead end unless otherwise +* annotated, because GCC 7 inserts it for certain +* divide-by-zero cases. +*/ +if (insn->type ==INSN_BUG ) + insn->dead_end = true; +hash_add (file->insn_hash, &insn->hash,sec_offset_hash (sec, insn->offset));list_add_tail (&insn->list, &file->insn_list); nr_insns++;
@@ -524,14 +532,6 @@ static int add_dead_ends(struct objtool_file *file)struct instruction *insn;/* -* By default, "ud2" is a dead end unless otherwise annotated, because -* GCC 7 inserts it for certain divide-by-zero cases. -*/ -for_each_insn (file, insn) -if (insn->type ==INSN_BUG ) - insn->dead_end = true; - -/* * Check for manually annotated dead ends. */ sec =find_section_by_name (file->elf,".rela.discard.unreachable" );
@@ -1114,6 +1114,9 @@ static void annotate_call_site(struct objtool_file *file,list_add_tail (&insn->call_node, &file->mcount_loc_list);return ; } + +if (!sibling &&dead_end_function (file, sym)) + insn->dead_end = true; } static void add_call_dest(struct objtool_file *file, struct instruction *insn,
@@ -2089,10 +2092,6 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; - ret =add_dead_ends (file); -if (ret) -return ret; -add_ignores (file);add_uaccess_safe (file);
@@ -2131,6 +2130,14 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; +/* +* Must be after add_call_destinations() such that it can override +* dead_end_function() marks. +*/ + ret =add_dead_ends (file); +if (ret) +return ret; + ret =add_jump_table_alts (file);if (ret)return ret;
@@ -3138,7 +3145,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,return 1 ; } -if (dead_end_function (file, insn->call_dest)) +if (insn->dead_end)return 0 ;break ;
Peter Zijlstra
maintainerPATCH v4 39/45 x86: Annotate call_on_stack()
In reply to Peter Zijlstravmlinux.o: warning: objtool: page_fault_oops()+0x13c: unreachable instruction
0000 000000000005b460 <page_fault_oops>:
...
0128 5b588: 49 89 23 mov %rsp,(%r11) 012b 5b58b: 4c 89 dc mov %r11,%rsp 012e 5b58e: 4c 89 f2 mov %r14,%rdx 0131 5b591: 48 89 ee mov %rbp,%rsi 0134 5b594: 4c 89 e7 mov %r12,%rdi 0137 5b597: e8 00 00 00 00 call 5b59c <page_fault_oops+0x13c> 5b598: R_X86_64_PLT32 handle_stack_overflow-0x4 013c 5b59c: 5c pop %rsp
vmlinux.o: warning: objtool: sysvec_reboot()+0x6d: unreachable instruction
0000 00000000000033f0 <sysvec_reboot>:
...
005d 344d: 4c 89 dc mov %r11,%rsp 0060 3450: e8 00 00 00 00 call 3455 <sysvec_reboot+0x65> 3451: R_X86_64_PLT32 irq_enter_rcu-0x4 0065 3455: 48 89 ef mov %rbp,%rdi 0068 3458: e8 00 00 00 00 call 345d <sysvec_reboot+0x6d> 3459: R_X86_64_PC32 .text+0x47d0c 006d 345d: e8 00 00 00 00 call 3462 <sysvec_reboot+0x72> 345e: R_X86_64_PLT32 irq_exit_rcu-0x4 0072 3462: 5c pop %rsp
Both cases are due to a call_on_stack() calling a __noreturn function. Since that's an inline asm, GCC can't do anything about the instructions after the CALL. Therefore put in an explicit ASM_REACHABLE annotation to make sure objtool and gcc are consistently confused about control flow.
--- arch/x86/include/asm/irq_stack.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/irq_stack.h +2 -1
@@ -99,7 +99,8 @@ }#define ASM_CALL_ARG0 \ - "call %P[__func] \n" + "call %P[__func] \n" \ + ASM_REACHABLE#define ASM_CALL_ARG1 \ "movq %[arg1], %%rdi \n" \
[tip: x86/core] x86: Annotate call_on_stack()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2170221418e44b3c3d0171194915156db31aebc1 Gitweb: https://git.kernel.org/tip/2170221418e44b3c3d0171194915156db31aebc1 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:50 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:37 +01:00
x86: Annotate call_on_stack()
vmlinux.o: warning: objtool: page_fault_oops()+0x13c: unreachable instruction
0000 000000000005b460 <page_fault_oops>:
...
0128 5b588: 49 89 23 mov %rsp,(%r11) 012b 5b58b: 4c 89 dc mov %r11,%rsp 012e 5b58e: 4c 89 f2 mov %r14,%rdx 0131 5b591: 48 89 ee mov %rbp,%rsi 0134 5b594: 4c 89 e7 mov %r12,%rdi 0137 5b597: e8 00 00 00 00 call 5b59c <page_fault_oops+0x13c> 5b598: R_X86_64_PLT32 handle_stack_overflow-0x4 013c 5b59c: 5c pop %rsp
vmlinux.o: warning: objtool: sysvec_reboot()+0x6d: unreachable instruction
0000 00000000000033f0 <sysvec_reboot>:
...
005d 344d: 4c 89 dc mov %r11,%rsp 0060 3450: e8 00 00 00 00 call 3455 <sysvec_reboot+0x65> 3451: R_X86_64_PLT32 irq_enter_rcu-0x4 0065 3455: 48 89 ef mov %rbp,%rdi 0068 3458: e8 00 00 00 00 call 345d <sysvec_reboot+0x6d> 3459: R_X86_64_PC32 .text+0x47d0c 006d 345d: e8 00 00 00 00 call 3462 <sysvec_reboot+0x72> 345e: R_X86_64_PLT32 irq_exit_rcu-0x4 0072 3462: 5c pop %rsp
Both cases are due to a call_on_stack() calling a __noreturn function. Since that's an inline asm, GCC can't do anything about the instructions after the CALL. Therefore put in an explicit ASM_REACHABLE annotation to make sure objtool and gcc are consistently confused about control flow.
--- arch/x86/include/asm/irq_stack.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/irq_stack.h +2 -1
@@ -99,7 +99,8 @@ }#define ASM_CALL_ARG0 \ - "call %P[__func] \n" + "call %P[__func] \n" \ + ASM_REACHABLE#define ASM_CALL_ARG1 \ "movq %[arg1], %%rdi \n" \
[tip: x86/core] x86: Annotate call_on_stack()
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: be0075951fde739f14ee2b659e2fd6e2499c46c0 Gitweb: https://git.kernel.org/tip/be0075951fde739f14ee2b659e2fd6e2499c46c0 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:50 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:44 +01:00
x86: Annotate call_on_stack()
vmlinux.o: warning: objtool: page_fault_oops()+0x13c: unreachable instruction
0000 000000000005b460 <page_fault_oops>:
...
0128 5b588: 49 89 23 mov %rsp,(%r11) 012b 5b58b: 4c 89 dc mov %r11,%rsp 012e 5b58e: 4c 89 f2 mov %r14,%rdx 0131 5b591: 48 89 ee mov %rbp,%rsi 0134 5b594: 4c 89 e7 mov %r12,%rdi 0137 5b597: e8 00 00 00 00 call 5b59c <page_fault_oops+0x13c> 5b598: R_X86_64_PLT32 handle_stack_overflow-0x4 013c 5b59c: 5c pop %rsp
vmlinux.o: warning: objtool: sysvec_reboot()+0x6d: unreachable instruction
0000 00000000000033f0 <sysvec_reboot>:
...
005d 344d: 4c 89 dc mov %r11,%rsp 0060 3450: e8 00 00 00 00 call 3455 <sysvec_reboot+0x65> 3451: R_X86_64_PLT32 irq_enter_rcu-0x4 0065 3455: 48 89 ef mov %rbp,%rdi 0068 3458: e8 00 00 00 00 call 345d <sysvec_reboot+0x6d> 3459: R_X86_64_PC32 .text+0x47d0c 006d 345d: e8 00 00 00 00 call 3462 <sysvec_reboot+0x72> 345e: R_X86_64_PLT32 irq_exit_rcu-0x4 0072 3462: 5c pop %rsp
Both cases are due to a call_on_stack() calling a __noreturn function. Since that's an inline asm, GCC can't do anything about the instructions after the CALL. Therefore put in an explicit ASM_REACHABLE annotation to make sure objtool and gcc are consistently confused about control flow.
--- arch/x86/include/asm/irq_stack.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/include/asm/irq_stack.h +2 -1
@@ -99,7 +99,8 @@ }#define ASM_CALL_ARG0 \ - "call %P[__func] \n" + "call %P[__func] \n" \ + ASM_REACHABLE#define ASM_CALL_ARG1 \ "movq %[arg1], %%rdi \n" \
Peter Zijlstra
maintainerPATCH v4 40/45 Kbuild: Allow whole module objtool runs
In reply to Peter ZijlstraJust like we have vmlinux.o objtool runs, add the ability to do whole module objtool runs.
--- scripts/Makefile.build | 44 ++------------------------------------------ scripts/Makefile.lib | 45 +++++++++++++++++++++++++++++++++++++++++++++ scripts/Makefile.modfinal | 1 + 3 files changed, 48 insertions(+), 42 deletions(-)
scripts/Makefile.build +2 -42
@@ -223,41 +223,6 @@ cmd_record_mcount = $(if $(findstring $( $(sub_cmd_record_mcount)) endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT -ifdef CONFIG_STACK_VALIDATION - -objtool := $(objtree)/tools/objtool/objtool - -objtool_args = \ - $(if $(CONFIG_UNWINDER_ORC),orc generate,check) \ - $(if $(part-of-module), --module) \ - $(if $(CONFIG_FRAME_POINTER),, --no-fp) \ - $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\ - $(if $(CONFIG_RETPOLINE), --retpoline) \ - $(if $(CONFIG_X86_SMAP), --uaccess) \ - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ - $(if $(CONFIG_SLS), --sls) - -cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) -cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd) - -endif # CONFIG_STACK_VALIDATION - -ifdef CONFIG_LTO_CLANG - -# Skip objtool for LLVM bitcode -$(obj)/%.o: objtool-enabled := - -else - -# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory -# 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file -# 'OBJECT_FILES_NON_STANDARD_foo.o := 'n': override directory skip for a file - -$(obj)/%.o: objtool-enabled = $(if $(filter-out y%, \ - $(OBJECT_FILES_NON_STANDARD_$(basetarget).o)$(OBJECT_FILES_NON_STANDARD)n),y) - -endif - ifdef CONFIG_TRIM_UNUSED_KSYMS cmd_gen_ksymdeps = \ $(CONFIG_SHELL) $(srctree)/scripts/gen_ksymdeps.sh $@ >> $(dot-target).cmd
@@ -292,21 +257,16 @@ ifdef CONFIG_LTO_CLANG # Module .o files may contain LLVM bitcode, compile them into native code # before ELF processing quiet_cmd_cc_lto_link_modules = LTO [M] $@ -cmd_cc_lto_link_modules = \ + cmd_cc_lto_link_modules = \ $(LD) $(ld_flags) -r -o $@ \ $(shell [ -s $(@:.lto.o=.o.symversions) ] && \ echo -T $(@:.lto.o=.o.symversions)) \ --whole-archive $(filter-out FORCE,$^) \ $(cmd_objtool) - -# objtool was skipped for LLVM bitcode, run it now that we have compiled -# modules into native code -$(obj)/%.lto.o: objtool-enabled = y -$(obj)/%.lto.o: part-of-module := y +endif $(obj)/%.lto.o: $(obj)/%.o FORCE $(call if_changed,cc_lto_link_modules) -endif cmd_mod = { \ echo $(if $($*-objs)$($*-y)$($*-m), $(addprefix $(obj)/, $($*-objs) $($*-y) $($*-m)), $(@:.mod=.o)); \
scripts/Makefile.lib +45 -0
48 lines not on this page · raw message
scripts/Makefile.modfinal +1 -0
@@ -32,6 +32,7 @@ ARCH_POSTLINK := $(wildcard $(srctree)/a quiet_cmd_ld_ko_o = LD [M] $@ cmd_ld_ko_o += \ + $(cmd_objtool_mod) \ $(LD) -r $(KBUILD_LDFLAGS) \ $(KBUILD_LDFLAGS_MODULE) $(LDFLAGS_MODULE) \ -T scripts/module.lds -o $@ $(filter %.o, $^); \
[tip: x86/core] Kbuild: Allow whole module objtool runs
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 8856dadf7ad3a0115b904d15532d2ed20c6985bb Gitweb: https://git.kernel.org/tip/8856dadf7ad3a0115b904d15532d2ed20c6985bb Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:51 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:38 +01:00
Kbuild: Allow whole module objtool runs
Just like we have vmlinux.o objtool runs, add the ability to do whole module objtool runs.
--- scripts/Makefile.build | 44 +------------------------------------ scripts/Makefile.lib | 45 ++++++++++++++++++++++++++++++++++++++- scripts/Makefile.modfinal | 1 +- 3 files changed, 48 insertions(+), 42 deletions(-)
scripts/Makefile.build +2 -42
@@ -223,41 +223,6 @@ cmd_record_mcount = $(if $(findstring $(strip $(CC_FLAGS_FTRACE)),$(_c_flags)), $(sub_cmd_record_mcount)) endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT -ifdef CONFIG_STACK_VALIDATION - -objtool := $(objtree)/tools/objtool/objtool - -objtool_args = \ - $(if $(CONFIG_UNWINDER_ORC),orc generate,check) \ - $(if $(part-of-module), --module) \ - $(if $(CONFIG_FRAME_POINTER),, --no-fp) \ - $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\ - $(if $(CONFIG_RETPOLINE), --retpoline) \ - $(if $(CONFIG_X86_SMAP), --uaccess) \ - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ - $(if $(CONFIG_SLS), --sls) - -cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) -cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd) - -endif # CONFIG_STACK_VALIDATION - -ifdef CONFIG_LTO_CLANG - -# Skip objtool for LLVM bitcode -$(obj)/%.o: objtool-enabled := - -else - -# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory -# 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file -# 'OBJECT_FILES_NON_STANDARD_foo.o := 'n': override directory skip for a file - -$(obj)/%.o: objtool-enabled = $(if $(filter-out y%, \ - $(OBJECT_FILES_NON_STANDARD_$(basetarget).o)$(OBJECT_FILES_NON_STANDARD)n),y) - -endif - ifdef CONFIG_TRIM_UNUSED_KSYMS cmd_gen_ksymdeps = \ $(CONFIG_SHELL) $(srctree)/scripts/gen_ksymdeps.sh $@ >> $(dot-target).cmd
@@ -292,21 +257,16 @@ ifdef CONFIG_LTO_CLANG # Module .o files may contain LLVM bitcode, compile them into native code # before ELF processing quiet_cmd_cc_lto_link_modules = LTO [M] $@ -cmd_cc_lto_link_modules = \ + cmd_cc_lto_link_modules = \ $(LD) $(ld_flags) -r -o $@ \ $(shell [ -s $(@:.lto.o=.o.symversions) ] && \ echo -T $(@:.lto.o=.o.symversions)) \ --whole-archive $(filter-out FORCE,$^) \ $(cmd_objtool) - -# objtool was skipped for LLVM bitcode, run it now that we have compiled -# modules into native code -$(obj)/%.lto.o: objtool-enabled = y -$(obj)/%.lto.o: part-of-module := y +endif $(obj)/%.lto.o: $(obj)/%.o FORCE $(call if_changed,cc_lto_link_modules) -endif cmd_mod = { \ echo $(if $($*-objs)$($*-y)$($*-m), $(addprefix $(obj)/, $($*-objs) $($*-y) $($*-m)), $(@:.mod=.o)); \
scripts/Makefile.lib +45 -0
48 lines not on this page · raw message
scripts/Makefile.modfinal +1 -0
@@ -32,6 +32,7 @@ ARCH_POSTLINK := $(wildcard $(srctree)/arch/$(SRCARCH)/Makefile.postlink) quiet_cmd_ld_ko_o = LD [M] $@ cmd_ld_ko_o += \ + $(cmd_objtool_mod) \ $(LD) -r $(KBUILD_LDFLAGS) \ $(KBUILD_LDFLAGS_MODULE) $(LDFLAGS_MODULE) \ -T scripts/module.lds -o $@ $(filter %.o, $^); \
Peter Zijlstra
maintainerPATCH v4 41/45 objtool: Read the NOENDBR annotation
In reply to Peter ZijlstraRead the new NOENDBR annotation. While there, attempt to not bloat struct instruction.
--- tools/objtool/check.c | 27 +++++++++++++++++++++++++++ tools/objtool/include/objtool/check.h | 13 ++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-)
tools/objtool/check.c +27 -0
@@ -1860,6 +1860,29 @@ static int read_unwind_hints(struct objtreturn 0 ; } +static int read_noendbr_hints (struct objtool_file *file) +{ +struct section *sec; +struct instruction *insn; +struct reloc *reloc; + + sec =find_section_by_name (file->elf,".rela.discard.noendbr" ); +if (!sec) +return 0 ; + +list_for_each_entry (reloc, &sec->reloc_list, list) { + insn =find_insn (file, reloc->sym->sec, reloc->sym->offset + reloc->addend); +if (!insn) { +WARN ("bad .discard.noendbr entry" ); +return -1 ; + } + + insn->noendbr =1 ; + } + +return 0 ; +} +static int read_retpoline_hints (struct objtool_file *file) {struct section *sec;
@@ -2097,6 +2120,10 @@ static int decode_sections(struct objtooif (ret)return ret; + ret =read_noendbr_hints (file); +if (ret) +return ret; +/* * Must be before add_{jump_call}_destination. */
tools/objtool/include/objtool/check.h +10 -3
@@ -45,11 +45,18 @@ struct instruction {unsigned int len;enum insn_type type;unsigned long immediate; -bool dead_end, ignore, ignore_alts; -bool hint; -bool retpoline_safe; + + u8 dead_end : 1, + ignore : 1, + ignore_alts : 1, + hint : 1, + retpoline_safe : 1, + noendbr : 1; +/* 2 bit hole */ s8 instr;u8 visited; +/* u8 hole */ +struct alt_group *alt_group;struct symbol *call_dest;struct instruction *jump_dest;
[tip: x86/core] objtool: Read the NOENDBR annotation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 51727f8e4a1ae15770ac30f05a90536c4aa3b2d8 Gitweb: https://git.kernel.org/tip/51727f8e4a1ae15770ac30f05a90536c4aa3b2d8 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:52 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:38 +01:00
objtool: Read the NOENDBR annotation
Read the new NOENDBR annotation. While there, attempt to not bloat struct instruction.
--- tools/objtool/check.c | 27 ++++++++++++++++++++++++++- tools/objtool/include/objtool/check.h | 13 ++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-)
tools/objtool/check.c +27 -0
@@ -1865,6 +1865,29 @@ static int read_unwind_hints(struct objtool_file *file)return 0 ; } +static int read_noendbr_hints (struct objtool_file *file) +{ +struct section *sec; +struct instruction *insn; +struct reloc *reloc; + + sec =find_section_by_name (file->elf,".rela.discard.noendbr" ); +if (!sec) +return 0 ; + +list_for_each_entry (reloc, &sec->reloc_list, list) { + insn =find_insn (file, reloc->sym->sec, reloc->sym->offset + reloc->addend); +if (!insn) { +WARN ("bad .discard.noendbr entry" ); +return -1 ; + } + + insn->noendbr =1 ; + } + +return 0 ; +} +static int read_retpoline_hints (struct objtool_file *file) {struct section *sec;
@@ -2098,6 +2121,10 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; + ret =read_noendbr_hints (file); +if (ret) +return ret; +/* * Must be before add_{jump_call}_destination. */
tools/objtool/include/objtool/check.h +10 -3
@@ -45,11 +45,18 @@ struct instruction {unsigned int len;enum insn_type type;unsigned long immediate; -bool dead_end, ignore, ignore_alts; -bool hint; -bool retpoline_safe; + + u8 dead_end : 1, + ignore : 1, + ignore_alts : 1, + hint : 1, + retpoline_safe : 1, + noendbr : 1; +/* 2 bit hole */ s8 instr;u8 visited; +/* u8 hole */ +struct alt_group *alt_group;struct symbol *call_dest;struct instruction *jump_dest;
[tip: x86/core] objtool: Read the NOENDBR annotation
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 96db4a988d653a7f18b518c25367f7bf238f4667 Gitweb: https://git.kernel.org/tip/96db4a988d653a7f18b518c25367f7bf238f4667 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:52 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:46 +01:00
objtool: Read the NOENDBR annotation
Read the new NOENDBR annotation. While there, attempt to not bloat struct instruction.
--- tools/objtool/check.c | 27 ++++++++++++++++++++++++++- tools/objtool/include/objtool/check.h | 13 ++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-)
tools/objtool/check.c +27 -0
@@ -1866,6 +1866,29 @@ static int read_unwind_hints(struct objtool_file *file)return 0 ; } +static int read_noendbr_hints (struct objtool_file *file) +{ +struct section *sec; +struct instruction *insn; +struct reloc *reloc; + + sec =find_section_by_name (file->elf,".rela.discard.noendbr" ); +if (!sec) +return 0 ; + +list_for_each_entry (reloc, &sec->reloc_list, list) { + insn =find_insn (file, reloc->sym->sec, reloc->sym->offset + reloc->addend); +if (!insn) { +WARN ("bad .discard.noendbr entry" ); +return -1 ; + } + + insn->noendbr =1 ; + } + +return 0 ; +} +static int read_retpoline_hints (struct objtool_file *file) {struct section *sec;
@@ -2099,6 +2122,10 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; + ret =read_noendbr_hints (file); +if (ret) +return ret; +/* * Must be before add_{jump_call}_destination. */
tools/objtool/include/objtool/check.h +10 -3
@@ -45,11 +45,18 @@ struct instruction {unsigned int len;enum insn_type type;unsigned long immediate; -bool dead_end, ignore, ignore_alts; -bool hint; -bool retpoline_safe; + + u8 dead_end : 1, + ignore : 1, + ignore_alts : 1, + hint : 1, + retpoline_safe : 1, + noendbr : 1; +/* 2 bit hole */ s8 instr;u8 visited; +/* u8 hole */ +struct alt_group *alt_group;struct symbol *call_dest;struct instruction *jump_dest;
Peter Zijlstra
maintainerPATCH v4 42/45 objtool: Add IBT/ENDBR decoding
In reply to Peter ZijlstraIntel IBT requires the target of any indirect CALL or JMP instruction to be the ENDBR instruction; optionally it allows those two instructions to have a NOTRACK prefix in order to avoid this requirement.
The kernel will not enable the use of NOTRACK, as such any occurence of it in compiler generated code should be flagged.
Teach objtool to Decode ENDBR instructions and WARN about NOTRACK prefixes.
--- tools/objtool/arch/x86/decode.c | 34 +++++++++++++++++++++++++++++----- tools/objtool/include/objtool/arch.h | 1 + 2 files changed, 30 insertions(+), 5 deletions(-)
tools/objtool/arch/x86/decode.c +29 -5
@@ -103,6 +103,18 @@ unsigned long arch_jump_destination(stru#define rm_is_mem (reg) (mod_is_mem() && !is_RIP() && rm_is(reg))#define rm_is_reg (reg) (mod_is_reg() && modrm_rm == (reg)) +static bool has_notrack_prefix (struct insn *insn) +{ +int i; + +for (i =0 ; i < insn->prefixes.nbytes; i++) { +if (insn->prefixes.bytes[i] ==0x3e ) +return true; + } + +return false; +} + int arch_decode_instruction(struct objtool_file *file, const struct section *sec, unsigned long offset, unsigned int maxlen, unsigned int *len, enum insn_type *type,
@@ -112,7 +124,7 @@ int arch_decode_instruction(struct objtoconst struct elf *elf = file->elf;struct insn insn;int x86_64, ret; -unsigned char op1, op2, op3, +unsigned char op1, op2, op3, prefix, rex =0 , rex_b =0 , rex_r =0 , rex_w =0 , rex_x =0 , modrm =0 , modrm_mod =0 , modrm_rm =0 , modrm_reg =0 , sib =0 ,/* sib_scale = 0, */ sib_index =0 , sib_base =0 ;
@@ -137,6 +149,8 @@ int arch_decode_instruction(struct objtoif (insn.vex_prefix.nbytes)return 0 ; + prefix = insn.prefixes.bytes[0 ]; + op1 = insn.opcode.bytes[0 ]; op2 = insn.opcode.bytes[1 ]; op3 = insn.opcode.bytes[2 ];
@@ -492,6 +506,12 @@ int arch_decode_instruction(struct objto/* nopl/nopw */ *type =INSN_NOP ; + }else if (op2 ==0x1e ) { + +if (prefix ==0xf3 && (modrm ==0xfa || modrm ==0xfb )) + *type =INSN_ENDBR ; + + }else if (op2 ==0x38 && op3 ==0xf8 ) {if (insn.prefixes.nbytes ==1 && insn.prefixes.bytes[0 ] ==0xf2 ) {
@@ -636,20 +656,24 @@ int arch_decode_instruction(struct objtobreak ;case 0xff : -if (modrm_reg ==2 || modrm_reg ==3 ) +if (modrm_reg ==2 || modrm_reg ==3 ) { *type =INSN_CALL_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==4 ) + }else if (modrm_reg ==4 ) { *type =INSN_JUMP_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==5 ) + }else if (modrm_reg ==5 ) {/* jmpf */ *type =INSN_CONTEXT_SWITCH ; -else if (modrm_reg ==6 ) { + }else if (modrm_reg ==6 ) {/* push from mem */ ADD_OP (op) {
tools/objtool/include/objtool/arch.h +1 -0
@@ -27,6 +27,7 @@ enum insn_type {INSN_STD ,INSN_CLD ,INSN_TRAP , +INSN_ENDBR , INSN_OTHER, };
[tip: x86/core] objtool: Add IBT/ENDBR decoding
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: df280fcb49f91de003afcac7b4da806d3ded52d6 Gitweb: https://git.kernel.org/tip/df280fcb49f91de003afcac7b4da806d3ded52d6 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:53 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 08 Mar 2022 23:53:38 +01:00
objtool: Add IBT/ENDBR decoding
Intel IBT requires the target of any indirect CALL or JMP instruction to be the ENDBR instruction; optionally it allows those two instructions to have a NOTRACK prefix in order to avoid this requirement.
The kernel will not enable the use of NOTRACK, as such any occurence of it in compiler generated code should be flagged.
Teach objtool to Decode ENDBR instructions and WARN about NOTRACK prefixes.
--- tools/objtool/arch/x86/decode.c | 34 +++++++++++++++++++++++---- tools/objtool/include/objtool/arch.h | 1 +- 2 files changed, 30 insertions(+), 5 deletions(-)
tools/objtool/arch/x86/decode.c +29 -5
@@ -103,6 +103,18 @@ unsigned long arch_jump_destination(struct instruction *insn)#define rm_is_mem (reg) (mod_is_mem() && !is_RIP() && rm_is(reg))#define rm_is_reg (reg) (mod_is_reg() && modrm_rm == (reg)) +static bool has_notrack_prefix (struct insn *insn) +{ +int i; + +for (i =0 ; i < insn->prefixes.nbytes; i++) { +if (insn->prefixes.bytes[i] ==0x3e ) +return true; + } + +return false; +} + int arch_decode_instruction(struct objtool_file *file, const struct section *sec, unsigned long offset, unsigned int maxlen, unsigned int *len, enum insn_type *type,
@@ -112,7 +124,7 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secconst struct elf *elf = file->elf;struct insn insn;int x86_64, ret; -unsigned char op1, op2, op3, +unsigned char op1, op2, op3, prefix, rex =0 , rex_b =0 , rex_r =0 , rex_w =0 , rex_x =0 , modrm =0 , modrm_mod =0 , modrm_rm =0 , modrm_reg =0 , sib =0 ,/* sib_scale = 0, */ sib_index =0 , sib_base =0 ;
@@ -137,6 +149,8 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secif (insn.vex_prefix.nbytes)return 0 ; + prefix = insn.prefixes.bytes[0 ]; + op1 = insn.opcode.bytes[0 ]; op2 = insn.opcode.bytes[1 ]; op3 = insn.opcode.bytes[2 ];
@@ -492,6 +506,12 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec/* nopl/nopw */ *type =INSN_NOP ; + }else if (op2 ==0x1e ) { + +if (prefix ==0xf3 && (modrm ==0xfa || modrm ==0xfb )) + *type =INSN_ENDBR ; + + }else if (op2 ==0x38 && op3 ==0xf8 ) {if (insn.prefixes.nbytes ==1 && insn.prefixes.bytes[0 ] ==0xf2 ) {
@@ -636,20 +656,24 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secbreak ;case 0xff : -if (modrm_reg ==2 || modrm_reg ==3 ) +if (modrm_reg ==2 || modrm_reg ==3 ) { *type =INSN_CALL_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==4 ) + }else if (modrm_reg ==4 ) { *type =INSN_JUMP_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==5 ) + }else if (modrm_reg ==5 ) {/* jmpf */ *type =INSN_CONTEXT_SWITCH ; -else if (modrm_reg ==6 ) { + }else if (modrm_reg ==6 ) {/* push from mem */ ADD_OP (op) {
tools/objtool/include/objtool/arch.h +1 -0
@@ -27,6 +27,7 @@ enum insn_type {INSN_STD ,INSN_CLD ,INSN_TRAP , +INSN_ENDBR , INSN_OTHER, };
[tip: x86/core] objtool: Add IBT/ENDBR decoding
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 7d209d13e7c3a3d60dc262f11a8ae4e6b4454d30 Gitweb: https://git.kernel.org/tip/7d209d13e7c3a3d60dc262f11a8ae4e6b4454d30 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 08 Mar 2022 16:30:53 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 15 Mar 2022 10:32:46 +01:00
objtool: Add IBT/ENDBR decoding
Intel IBT requires the target of any indirect CALL or JMP instruction to be the ENDBR instruction; optionally it allows those two instructions to have a NOTRACK prefix in order to avoid this requirement.
The kernel will not enable the use of NOTRACK, as such any occurence of it in compiler generated code should be flagged.
Teach objtool to Decode ENDBR instructions and WARN about NOTRACK prefixes.
--- tools/objtool/arch/x86/decode.c | 34 +++++++++++++++++++++++---- tools/objtool/include/objtool/arch.h | 1 +- 2 files changed, 30 insertions(+), 5 deletions(-)
tools/objtool/arch/x86/decode.c +29 -5
@@ -103,6 +103,18 @@ unsigned long arch_jump_destination(struct instruction *insn)#define rm_is_mem (reg) (mod_is_mem() && !is_RIP() && rm_is(reg))#define rm_is_reg (reg) (mod_is_reg() && modrm_rm == (reg)) +static bool has_notrack_prefix (struct insn *insn) +{ +int i; + +for (i =0 ; i < insn->prefixes.nbytes; i++) { +if (insn->prefixes.bytes[i] ==0x3e ) +return true; + } + +return false; +} + int arch_decode_instruction(struct objtool_file *file, const struct section *sec, unsigned long offset, unsigned int maxlen, unsigned int *len, enum insn_type *type,
@@ -112,7 +124,7 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secconst struct elf *elf = file->elf;struct insn insn;int x86_64, ret; -unsigned char op1, op2, op3, +unsigned char op1, op2, op3, prefix, rex =0 , rex_b =0 , rex_r =0 , rex_w =0 , rex_x =0 , modrm =0 , modrm_mod =0 , modrm_rm =0 , modrm_reg =0 , sib =0 ,/* sib_scale = 0, */ sib_index =0 , sib_base =0 ;
@@ -137,6 +149,8 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secif (insn.vex_prefix.nbytes)return 0 ; + prefix = insn.prefixes.bytes[0 ]; + op1 = insn.opcode.bytes[0 ]; op2 = insn.opcode.bytes[1 ]; op3 = insn.opcode.bytes[2 ];
@@ -492,6 +506,12 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec/* nopl/nopw */ *type =INSN_NOP ; + }else if (op2 ==0x1e ) { + +if (prefix ==0xf3 && (modrm ==0xfa || modrm ==0xfb )) + *type =INSN_ENDBR ; + + }else if (op2 ==0x38 && op3 ==0xf8 ) {if (insn.prefixes.nbytes ==1 && insn.prefixes.bytes[0 ] ==0xf2 ) {
@@ -636,20 +656,24 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *secbreak ;case 0xff : -if (modrm_reg ==2 || modrm_reg ==3 ) +if (modrm_reg ==2 || modrm_reg ==3 ) { *type =INSN_CALL_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==4 ) + }else if (modrm_reg ==4 ) { *type =INSN_JUMP_DYNAMIC ; +if (has_notrack_prefix (&insn)) +WARN ("notrack prefix found at %s:0x%lx" , sec->name, offset); -else if (modrm_reg ==5 ) + }else if (modrm_reg ==5 ) {/* jmpf */ *type =INSN_CONTEXT_SWITCH ; -else if (modrm_reg ==6 ) { + }else if (modrm_reg ==6 ) {/* push from mem */ ADD_OP (op) {
tools/objtool/include/objtool/arch.h +1 -0
@@ -27,6 +27,7 @@ enum insn_type {INSN_STD ,INSN_CLD ,INSN_TRAP , +INSN_ENDBR , INSN_OTHER, };