objtool: noinstr validation for static branches/calls
Subsystems (2)
Josh Poimboeuf
maintainerv2 cover objtool: noinstr validation for static branches/calls
v2:
- Add some fixes reported by kbuild.
FWIW, this reports the following static keys:
- sched_clock_running
- __sched_clock_stable
- mds_idle_clear
- vmx_l1d_flush_cond
- stack_erasing_bypass
and the following static calls:
- pv_sched_clock
- x86_idle
(not necessarily an exhaustive list, just the ones I've seen)
Josh Poimboeuf (3):
jump_label: Add annotations for validating noinstr usage
static_call: Add read-only-after-init static calls
objtool: Add noinstr validation for static branches/calls
include/linux/jump_label.h | 30 ++++++-- include/linux/objtool.h | 7 ++ include/linux/static_call.h | 19 +++++ tools/objtool/Documentation/objtool.txt | 34 +++++++++ tools/objtool/check.c | 92 ++++++++++++++++++++++--- tools/objtool/include/objtool/check.h | 1 + tools/objtool/include/objtool/elf.h | 1 + tools/objtool/include/objtool/special.h | 1 + tools/objtool/special.c | 18 ++++- 9 files changed, 187 insertions(+), 16 deletions(-)
Signature · 1 line
2.47.0
Josh Poimboeuf
maintainerPATCH v2 1/3 jump_label: Add annotations for validating noinstr usage
Deferring a code patching IPI is unsafe if the patched code is in a noinstr region. In that case the text poke code must trigger an immediate IPI to all CPUs, which can rudely interrupt an isolated NO_HZ CPU running in userspace.
Some noinstr static branches may really need to be patched at runtime, despite the resulting disruption. Add DEFINE_STATIC_KEY_*_NOINSTR() variants for those. They don't do anything special yet; that will come later.
--- include/linux/jump_label.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
include/linux/jump_label.h +17 -0
@@ -385,6 +385,23 @@ struct static_key_false {#define DEFINE_STATIC_KEY_FALSE_RO (name) \ struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT +/* +* The _NOINSTR variants are used to tell objtool the static key is allowed to +* be used in noinstr code. +* +* They should almost never be used, as they prevent code patching IPIs from +* being deferred, which can be problematic for isolated NOHZ_FULL CPUs running +* in pure userspace. +* +* If using one of these _NOINSTR variants, please add a comment above the +* definition with the rationale. +*/ +#define DEFINE_STATIC_KEY_TRUE_NOINSTR (name) \ + DEFINE_STATIC_KEY_TRUE(name) + +#define DEFINE_STATIC_KEY_FALSE_NOINSTR (name) \ + DEFINE_STATIC_KEY_FALSE(name) +#define DECLARE_STATIC_KEY_FALSE (name) \ extern struct static_key_false name
Signature · 1 line
2.47.0
Josh Poimboeuf
maintainerPATCH v2 2/3 static_call: Add read-only-after-init static calls
In reply to Josh PoimboeufDeferring a code patching IPI is unsafe if the patched code is in a noinstr region. In that case the text poke code must trigger an immediate IPI to all CPUs, which can rudely interrupt an isolated NO_HZ CPU running in userspace.
If a noinstr static call only needs to be patched during boot, its key can be made ro-after-init to ensure it will never be patched at runtime.
--- include/linux/static_call.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
include/linux/static_call.h +16 -0
@@ -190,6 +190,14 @@ extern long __static_call_return0(void); }; \ ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) +#define DEFINE_STATIC_CALL_RO (name, _func) \ + DECLARE_STATIC_CALL(name, _func); \ + struct static_call_key __ro_after_init STATIC_CALL_KEY(name) = {\ + .func = _func, \ + .type = 1, \ + }; \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) +#define DEFINE_STATIC_CALL_NULL (name, _func) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \
@@ -198,6 +206,14 @@ extern long __static_call_return0(void); }; \ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) +#define DEFINE_STATIC_CALL_NULL_RO (name, _func) \ + DECLARE_STATIC_CALL(name, _func); \ + struct static_call_key __ro_after_init STATIC_CALL_KEY(name) = {\ + .func = NULL, \ + .type = 1, \ + }; \ + ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) +#define DEFINE_STATIC_CALL_RET0 (name, _func) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \
Signature · 1 line
2.47.0
Josh Poimboeuf
maintainerPATCH v2 3/3 objtool: Add noinstr validation for static branches/calls
In reply to Josh PoimboeufWarn about static branches/calls in noinstr regions, unless the corresponding key is RO-after-init or has been manually whitelisted with DEFINE_STATIC_KEY_*_NOINSTR(().
--- include/linux/jump_label.h | 17 +++-- include/linux/objtool.h | 7 ++ include/linux/static_call.h | 3 + tools/objtool/Documentation/objtool.txt | 34 +++++++++ tools/objtool/check.c | 92 ++++++++++++++++++++++--- tools/objtool/include/objtool/check.h | 1 + tools/objtool/include/objtool/elf.h | 1 + tools/objtool/include/objtool/special.h | 1 + tools/objtool/special.c | 18 ++++- 9 files changed, 156 insertions(+), 18 deletions(-)
include/linux/jump_label.h +11 -6
@@ -75,6 +75,7 @@#include <linux/types.h> #include <linux/compiler.h> +#include <linux/objtool.h> extern bool static_key_initialized;
@@ -373,8 +374,9 @@ struct static_key_false {#define DEFINE_STATIC_KEY_TRUE (name) \ struct static_key_true name = STATIC_KEY_TRUE_INIT -#define DEFINE_STATIC_KEY_TRUE_RO (name) \ - struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT +#define DEFINE_STATIC_KEY_TRUE_RO (name) \ + struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT; \ + ANNOTATE_NOINSTR_ALLOWED(name)#define DECLARE_STATIC_KEY_TRUE (name) \ extern struct static_key_true name
@@ -382,8 +384,9 @@ struct static_key_false {#define DEFINE_STATIC_KEY_FALSE (name) \ struct static_key_false name = STATIC_KEY_FALSE_INIT -#define DEFINE_STATIC_KEY_FALSE_RO (name) \ - struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT +#define DEFINE_STATIC_KEY_FALSE_RO (name) \ + struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT; \ + ANNOTATE_NOINSTR_ALLOWED(name)/* * The _NOINSTR variants are used to tell objtool the static key is allowed to
@@ -397,10 +400,12 @@ struct static_key_false {* definition with the rationale. */ #define DEFINE_STATIC_KEY_TRUE_NOINSTR (name) \ - DEFINE_STATIC_KEY_TRUE(name) + DEFINE_STATIC_KEY_TRUE(name); \ + ANNOTATE_NOINSTR_ALLOWED(name)#define DEFINE_STATIC_KEY_FALSE_NOINSTR (name) \ - DEFINE_STATIC_KEY_FALSE(name) + DEFINE_STATIC_KEY_FALSE(name); \ + ANNOTATE_NOINSTR_ALLOWED(name)#define DECLARE_STATIC_KEY_FALSE (name) \ extern struct static_key_false name
include/linux/objtool.h +7 -0
@@ -34,6 +34,12 @@ static void __used __section(".discard.func_stack_frame_non_standard") \ *__func_stack_frame_non_standard_##func = func +#define __ANNOTATE_NOINSTR_ALLOWED (key) \ + static void __used __section(".discard.noinstr_allowed") \ + *__annotate_noinstr_allowed_##key = &key + +#define ANNOTATE_NOINSTR_ALLOWED (key) __ANNOTATE_NOINSTR_ALLOWED(key) +/* * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore * for the case where a function is intentionally missing frame pointer setup,
@@ -157,6 +163,7 @@#define STACK_FRAME_NON_STANDARD_FP (func)#define ANNOTATE_NOENDBR #define ASM_REACHABLE +#define ANNOTATE_NOINSTR_ALLOWED (key)#else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
include/linux/static_call.h +3 -0
@@ -133,6 +133,7 @@#include <linux/types.h> #include <linux/cpu.h> +#include <linux/objtool.h> #include <linux/static_call_types.h> #ifdef CONFIG_HAVE_STATIC_CALL
@@ -196,6 +197,7 @@ extern long __static_call_return0(void); .func = _func, \ .type = 1, \ }; \ +ANNOTATE_NOINSTR_ALLOWED (STATIC_CALL_TRAMP (name)); \ ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)#define DEFINE_STATIC_CALL_NULL (name, _func) \
@@ -212,6 +214,7 @@ extern long __static_call_return0(void); .func = NULL, \ .type = 1, \ }; \ +ANNOTATE_NOINSTR_ALLOWED (STATIC_CALL_TRAMP (name)); \ARCH_DEFINE_STATIC_CALL_NULL_TRAMP (name) #define DEFINE_STATIC_CALL_RET0(name, _func) \
tools/objtool/Documentation/objtool.txt +34 -0
@@ -447,6 +447,40 @@ the objtool maintainers. names and does not use module_init() / module_exit() macros to create them. +13. file.o: warning: func()+0x2a: key: non-RO static key usage in noinstr code + file.o: warning: func()+0x2a: key: non-RO static call usage in noinstr code + + This means that noinstr function func() uses a static key or + static call named 'key' which can be modified at runtime. This is + discouraged because it prevents code patching IPIs from being + deferred. + + You have the following options: + + 1) Check whether the static key/call in question is only modified + during init. If so, define it as read-only-after-init with + DEFINE_STATIC_KEY_*_RO() or DEFINE_STATIC_CALL_RO(). + + 2) Avoid the runtime patching. For static keys this can be done by + using static_key_enabled() or by getting rid of the static key + altogether if performance is not a concern. + + For static calls, something like the following could be done: + + target = static_call_query(foo); + if (target == func1) + func1(); + else if (target == func2) + func2(); + ... + + 3) Silence the warning by defining the static key/call with + DEFINE_STATIC_*_NOINSTR(). This decision should not + be taken lightly as it may result in code patching IPIs getting + sent to isolated NOHZ_FULL CPUs running in pure userspace. A + comment should be added above the definition explaining the + rationale for the decision. + If the error doesn't seem to make sense, it could be a bug in objtool. Feel free to ask the objtool maintainer for help.
tools/objtool/check.c +83 -9
146 lines not on this page · raw message
tools/objtool/include/objtool/check.h +1 -0
@@ -77,6 +77,7 @@ struct instruction {struct symbol *sym;struct stack_op *stack_ops;struct cfi_state *cfi; +struct symbol *key; };static inline struct symbol *insn_func (struct instruction *insn)
tools/objtool/include/objtool/elf.h +1 -0
@@ -69,6 +69,7 @@ struct symbol { u8 embedded_insn : 1; u8 local_label : 1; u8 frame_pointer : 1; + u8 noinstr_allowed : 1;struct list_head pv_target;struct reloc *relocs; };
tools/objtool/include/objtool/special.h +1 -0
@@ -20,6 +20,7 @@ struct special_alt {bool skip_alt;bool jump_or_nop;u8 key_addend; +struct symbol *key;struct section *orig_sec;unsigned long orig_off;
tools/objtool/special.c +15 -3
@@ -119,14 +119,26 @@ static int get_alt_entry(struct elf *elf, const struct special_entry *entry,if (entry->key) {struct reloc *key_reloc; +struct symbol *key; +s64 key_addend; key_reloc =find_reloc_by_dest (elf, sec, offset + entry->key);if (!key_reloc) { -WARN_FUNC ("can't find key reloc" , - sec, offset + entry->key); +WARN_FUNC ("can't find key reloc" , sec, offset + entry->key);return -1 ; } - alt->key_addend =reloc_addend (key_reloc); + + key = key_reloc->sym; + key_addend =reloc_addend (key_reloc); + +if (key->type ==STT_SECTION ) + key =find_symbol_by_offset (key->sec, key_addend & ~3 ); + +/* embedded keys not supported */ +if (key) { + alt->key = key; + alt->key_addend = key_addend; + } }return 0 ;
Signature · 1 line
2.47.0
Quoted 12 lines · Josh Poimboeuf
v2:
- Add some fixes reported by kbuild.FWIW, this reports the following static keys:
-
sched_clock_running
-__sched_clock_stable
-mds_idle_clear
-vmx_l1d_flush_cond
-stack_erasing_bypass
Quoted 5 lines
and the following static calls:
-
pv_sched_clock
-x86_idle
And you event did the static calls, sweet! Thanks a ton Josh, lemme catch up with emails and then I'll look into these.
Josh Poimboeuf
maintainerQuoted 11 lines · Josh Poimboeuf
v2:
- Add some fixes reported by kbuild.FWIW, this reports the following static keys:
-
sched_clock_running
-__sched_clock_stable
-mds_idle_clear
-vmx_l1d_flush_cond
-stack_erasing_bypass
- perf_lopwr_cb
Quoted 4 lines
and the following static calls:
-
pv_sched_clock
-x86_idle
Signature · 1 line
Josh
Quoted 5 lines · Josh Poimboeuf
Quoted 4 lines · Josh Poimboeuf
v2:
- Add some fixes reported by kbuild.
Quoted 8 lines
Quoted 7 lines
FWIW, this reports the following static keys:
-
sched_clock_running
-__sched_clock_stable
-mds_idle_clear
-vmx_l1d_flush_cond
-stack_erasing_bypass
sched_clock_running, __sched_clock_stable and mds_idle_clear are already covered in v3
vmx_l1d_flush_cond is modified
- at init via __init vmx_init()
- via a module parameter write ending in vmentry_l1d_flush_set() So either init or user/admin action, IMO that's a "don't touch it" scenario.
stack_erasing_bypass is modified only via a sysctl write, so IMO same argument.
Quoted 5 lines
Quoted 4 lines
and the following static calls:
-
pv_sched_clock
-x86_idle-
perf_lopwr_cb
All of these are only ever modified at __init, so they can be made __ro_after_init.
Quoted 3 lines
Signature · 2 lines
-- Josh
Re: [PATCH v2 3/3] objtool: Add noinstr validation for static branches/calls
In reply to Josh PoimboeufQuoted 10 lines · Josh Poimboeuf
tools/objtool/check.c +5
-0@@ -3532,14 +3577,29 @@ static inline bool noinstr_call_dest(struct objtool_file *file,return false; } +static char *static_call_name (struct symbol *func) +{ +return func->name +strlen ("__SCT__" ); +} + static int validate_call(struct objtool_file *file,
tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3591
Quoted 10 lines
tools/objtool/check.c +3
-4struct instruction *insn, struct insn_state *state) { -if (state->noinstr && state->instr <=0 && - !noinstr_call_dest (file, insn,insn_call_dest (insn))) { -WARN_INSN (insn,"call to %s() leaves .noinstr.text section" ,call_dest_name (insn)); -return 1 ; +if (state->noinstr && state->instr <=0 ) { +struct symbol *dest =insn_call_dest (insn); +
Interestingly only after I tried making __flush_tlb_all() noinstr did this start causing a segfault due to a NULL dest. I added a NULL check here but I wonder if I'm not just plastering over some other issue?
Quoted 14 lines
tools/objtool/check.c +11
-0+if (dest->static_call_tramp) { +if (!dest->noinstr_allowed) { +WARN_INSN (insn,"%s: non-RO static call usage in noinstr" , +static_call_name (dest)); + } + + }else if (!noinstr_call_allowed (file, insn, dest)) { +WARN_INSN (insn,"call to %s() leaves .noinstr.text section" , +call_dest_name (insn)); +return 1 ; + } }if (state->uaccess && !func_uaccess_safe (insn_call_dest (insn))) {
Re: [PATCH v2 3/3] objtool: Add noinstr validation for static branches/calls
Quoted 26 lines · Valentin Schneider
Quoted 10 lines · Josh Poimboeuf
tools/objtool/check.c +5
-0@@ -3532,14 +3577,29 @@ static inline bool noinstr_call_dest(struct objtool_file *file,return false; } +static char *static_call_name (struct symbol *func) +{ +return func->name +strlen ("__SCT__" ); +} + static int validate_call(struct objtool_file *file,tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3591
Quoted 10 lines
tools/objtool/check.c +3
-4struct instruction *insn, struct insn_state *state) { -if (state->noinstr && state->instr <=0 && - !noinstr_call_dest (file, insn,insn_call_dest (insn))) { -WARN_INSN (insn,"call to %s() leaves .noinstr.text section" ,call_dest_name (insn)); -return 1 ; +if (state->noinstr && state->instr <=0 ) { +struct symbol *dest =insn_call_dest (insn); +Interestingly only after I tried making
__flush_tlb_all()noinstr did this start causing a segfault due to a NULL dest. I added a NULL check here but I wonder if I'm not just plastering over some other issue?
Quoted 1 line
tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3592
Quoted 1 line
tools/objtool/check.c +1
-0+if (dest->static_call_tramp) {
Yeah, this line can probably just be:
if (dest && dest->static_call_tramp)
Signature · 1 line
Josh