Skip to contentSkip to conversation

objtool: noinstr validation for static branches/calls

PATCH In review Josh Poimboeuf 3 participants 9 messages Nov 27, 2024 – Dec 13, 2024

Subsystems (2)
lore
Share & tools
lore

Josh Poimboeuf

author maintainer
#0 lore raw

v2 cover objtool: noinstr validation for static branches/calls

v2:
- Add some fixes reported by kbuild.

FWIW, this reports the following static keys:

- sched_clock_running
- __sched_clock_stable
- mds_idle_clear
- vmx_l1d_flush_cond
- stack_erasing_bypass

and the following static calls:

- pv_sched_clock
- x86_idle

(not necessarily an exhaustive list, just the ones I've seen)

Josh Poimboeuf (3):
jump_label: Add annotations for validating noinstr usage
static_call: Add read-only-after-init static calls
objtool: Add noinstr validation for static branches/calls

 include/linux/jump_label.h              | 30 ++++++--
 include/linux/objtool.h                 |  7 ++
 include/linux/static_call.h             | 19 +++++
 tools/objtool/Documentation/objtool.txt | 34 +++++++++
 tools/objtool/check.c                   | 92 ++++++++++++++++++++++---
 tools/objtool/include/objtool/check.h   |  1 +
 tools/objtool/include/objtool/elf.h     |  1 +
 tools/objtool/include/objtool/special.h |  1 +
 tools/objtool/special.c                 | 18 ++++-
 9 files changed, 187 insertions(+), 16 deletions(-)
Signature · 1 line
2.47.0
author maintainer
#1 lore raw

PATCH v2 1/3 jump_label: Add annotations for validating noinstr usage

Deferring a code patching IPI is unsafe if the patched code is in a noinstr region. In that case the text poke code must trigger an immediate IPI to all CPUs, which can rudely interrupt an isolated NO_HZ CPU running in userspace.

Some noinstr static branches may really need to be patched at runtime, despite the resulting disruption. Add DEFINE_STATIC_KEY_*_NOINSTR() variants for those. They don't do anything special yet; that will come later.

---
 include/linux/jump_label.h | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
include/linux/jump_label.h +17 -0
index f5a2727ca4a9..88bb6e32fdcb 100644
@@ -385,6 +385,23 @@ struct static_key_false {
 #define DEFINE_STATIC_KEY_FALSE_RO(name)	\
 	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
 
+/*
+ * The _NOINSTR variants are used to tell objtool the static key is allowed to
+ * be used in noinstr code.
+ *
+ * They should almost never be used, as they prevent code patching IPIs from
+ * being deferred, which can be problematic for isolated NOHZ_FULL CPUs running
+ * in pure userspace.
+ *
+ * If using one of these _NOINSTR variants, please add a comment above the
+ * definition with the rationale.
+ */
+#define DEFINE_STATIC_KEY_TRUE_NOINSTR(name)					\
+	DEFINE_STATIC_KEY_TRUE(name)
+
+#define DEFINE_STATIC_KEY_FALSE_NOINSTR(name)					\
+	DEFINE_STATIC_KEY_FALSE(name)
+
 #define DECLARE_STATIC_KEY_FALSE(name)	\
 	extern struct static_key_false name
 
Signature · 1 line
2.47.0
author maintainer
#2 lore raw

PATCH v2 2/3 static_call: Add read-only-after-init static calls

In reply to Josh Poimboeuf

Deferring a code patching IPI is unsafe if the patched code is in a noinstr region. In that case the text poke code must trigger an immediate IPI to all CPUs, which can rudely interrupt an isolated NO_HZ CPU running in userspace.

If a noinstr static call only needs to be patched during boot, its key can be made ro-after-init to ensure it will never be patched at runtime.

---
 include/linux/static_call.h | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
include/linux/static_call.h +16 -0
index 141e6b176a1b..34970e178fdf 100644
@@ -190,6 +190,14 @@ extern long __static_call_return0(void);
 	};								\
 	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 
+#define DEFINE_STATIC_CALL_RO(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key __ro_after_init STATIC_CALL_KEY(name) = {\
+		.func = _func,						\
+		.type = 1,						\
+	};								\
+	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
 #define DEFINE_STATIC_CALL_NULL(name, _func)				\
 	DECLARE_STATIC_CALL(name, _func);				\
 	struct static_call_key STATIC_CALL_KEY(name) = {		\
@@ -198,6 +206,14 @@ extern long __static_call_return0(void);
 	};								\
 	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
 
+#define DEFINE_STATIC_CALL_NULL_RO(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key __ro_after_init STATIC_CALL_KEY(name) = {\
+		.func = NULL,						\
+		.type = 1,						\
+	};								\
+	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+
 #define DEFINE_STATIC_CALL_RET0(name, _func)				\
 	DECLARE_STATIC_CALL(name, _func);				\
 	struct static_call_key STATIC_CALL_KEY(name) = {		\
Signature · 1 line
2.47.0
author maintainer
#3 lore raw

PATCH v2 3/3 objtool: Add noinstr validation for static branches/calls

In reply to Josh Poimboeuf

Warn about static branches/calls in noinstr regions, unless the corresponding key is RO-after-init or has been manually whitelisted with DEFINE_STATIC_KEY_*_NOINSTR(().

---
 include/linux/jump_label.h              | 17 +++--
 include/linux/objtool.h                 |  7 ++
 include/linux/static_call.h             |  3 +
 tools/objtool/Documentation/objtool.txt | 34 +++++++++
 tools/objtool/check.c                   | 92 ++++++++++++++++++++++---
 tools/objtool/include/objtool/check.h   |  1 +
 tools/objtool/include/objtool/elf.h     |  1 +
 tools/objtool/include/objtool/special.h |  1 +
 tools/objtool/special.c                 | 18 ++++-
 9 files changed, 156 insertions(+), 18 deletions(-)
include/linux/jump_label.h +11 -6
index 88bb6e32fdcb..dc8a82a62c10 100644
@@ -75,6 +75,7 @@
 
 #include <linux/types.h>
 #include <linux/compiler.h>
+#include <linux/objtool.h>
 
 extern bool static_key_initialized;
 
@@ -373,8 +374,9 @@ struct static_key_false {
 #define DEFINE_STATIC_KEY_TRUE(name)	\
 	struct static_key_true name = STATIC_KEY_TRUE_INIT
 
-#define DEFINE_STATIC_KEY_TRUE_RO(name)	\
-	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
+#define DEFINE_STATIC_KEY_TRUE_RO(name)						\
+	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT;	\
+	ANNOTATE_NOINSTR_ALLOWED(name)
 
 #define DECLARE_STATIC_KEY_TRUE(name)	\
 	extern struct static_key_true name
@@ -382,8 +384,9 @@ struct static_key_false {
 #define DEFINE_STATIC_KEY_FALSE(name)	\
 	struct static_key_false name = STATIC_KEY_FALSE_INIT
 
-#define DEFINE_STATIC_KEY_FALSE_RO(name)	\
-	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
+#define DEFINE_STATIC_KEY_FALSE_RO(name)					\
+	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT;	\
+	ANNOTATE_NOINSTR_ALLOWED(name)
 
 /*
  * The _NOINSTR variants are used to tell objtool the static key is allowed to
@@ -397,10 +400,12 @@ struct static_key_false {
  * definition with the rationale.
  */
 #define DEFINE_STATIC_KEY_TRUE_NOINSTR(name)					\
-	DEFINE_STATIC_KEY_TRUE(name)
+	DEFINE_STATIC_KEY_TRUE(name);						\
+	ANNOTATE_NOINSTR_ALLOWED(name)
 
 #define DEFINE_STATIC_KEY_FALSE_NOINSTR(name)					\
-	DEFINE_STATIC_KEY_FALSE(name)
+	DEFINE_STATIC_KEY_FALSE(name);						\
+	ANNOTATE_NOINSTR_ALLOWED(name)
 
 #define DECLARE_STATIC_KEY_FALSE(name)	\
 	extern struct static_key_false name
include/linux/objtool.h +7 -0
index b3b8d3dab52d..1a7389f27306 100644
@@ -34,6 +34,12 @@
 	static void __used __section(".discard.func_stack_frame_non_standard") \
 		*__func_stack_frame_non_standard_##func = func
 
+#define __ANNOTATE_NOINSTR_ALLOWED(key) \
+	static void __used __section(".discard.noinstr_allowed") \
+		*__annotate_noinstr_allowed_##key = &key
+
+#define ANNOTATE_NOINSTR_ALLOWED(key) __ANNOTATE_NOINSTR_ALLOWED(key)
+
 /*
  * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore
  * for the case where a function is intentionally missing frame pointer setup,
@@ -157,6 +163,7 @@
 #define STACK_FRAME_NON_STANDARD_FP(func)
 #define ANNOTATE_NOENDBR
 #define ASM_REACHABLE
+#define ANNOTATE_NOINSTR_ALLOWED(key)
 #else
 #define ANNOTATE_INTRA_FUNCTION_CALL
 .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
include/linux/static_call.h +3 -0
index 34970e178fdf..c7648ed72361 100644
@@ -133,6 +133,7 @@
 
 #include <linux/types.h>
 #include <linux/cpu.h>
+#include <linux/objtool.h>
 #include <linux/static_call_types.h>
 
 #ifdef CONFIG_HAVE_STATIC_CALL
@@ -196,6 +197,7 @@ extern long __static_call_return0(void);
 		.func = _func,						\
 		.type = 1,						\
 	};								\
+	ANNOTATE_NOINSTR_ALLOWED(STATIC_CALL_TRAMP(name));		\
 	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 
 #define DEFINE_STATIC_CALL_NULL(name, _func)				\
@@ -212,6 +214,7 @@ extern long __static_call_return0(void);
 		.func = NULL,						\
 		.type = 1,						\
 	};								\
+	ANNOTATE_NOINSTR_ALLOWED(STATIC_CALL_TRAMP(name));		\
 	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
 
 #define DEFINE_STATIC_CALL_RET0(name, _func)				\
tools/objtool/Documentation/objtool.txt +34 -0
index 7c3ee959b63c..922d3b41541d 100644
@@ -447,6 +447,40 @@ the objtool maintainers.
    names and does not use module_init() / module_exit() macros to create
    them.
 
+13. file.o: warning: func()+0x2a: key: non-RO static key usage in noinstr code
+    file.o: warning: func()+0x2a: key: non-RO static call usage in noinstr code
+
+  This means that noinstr function func() uses a static key or
+  static call named 'key' which can be modified at runtime.  This is
+  discouraged because it prevents code patching IPIs from being
+  deferred.
+
+  You have the following options:
+
+  1) Check whether the static key/call in question is only modified
+     during init.  If so, define it as read-only-after-init with
+     DEFINE_STATIC_KEY_*_RO() or DEFINE_STATIC_CALL_RO().
+
+  2) Avoid the runtime patching.  For static keys this can be done by
+     using static_key_enabled() or by getting rid of the static key
+     altogether if performance is not a concern.
+
+     For static calls, something like the following could be done:
+
+       target = static_call_query(foo);
+       if (target == func1)
+	       func1();
+	else if (target == func2)
+		func2();
+	...
+
+  3) Silence the warning by defining the static key/call with
+     DEFINE_STATIC_*_NOINSTR().  This decision should not
+     be taken lightly as it may result in code patching IPIs getting
+     sent to isolated NOHZ_FULL CPUs running in pure userspace.  A
+     comment should be added above the definition explaining the
+     rationale for the decision.
+
 
 If the error doesn't seem to make sense, it could be a bug in objtool.
 Feel free to ask the objtool maintainer for help.
tools/objtool/check.c +83 -9

146 lines not on this page · raw message

tools/objtool/include/objtool/check.h +1 -0
index daa46f1f0965..c0da7246eac7 100644
@@ -77,6 +77,7 @@ struct instruction {
 	struct symbol *sym;
 	struct stack_op *stack_ops;
 	struct cfi_state *cfi;
+	struct symbol *key;
 };
 
 static inline struct symbol *insn_func(struct instruction *insn)
tools/objtool/include/objtool/elf.h +1 -0
index d7e815c2fd15..0cb79931262b 100644
@@ -69,6 +69,7 @@ struct symbol {
 	u8 embedded_insn     : 1;
 	u8 local_label       : 1;
 	u8 frame_pointer     : 1;
+	u8 noinstr_allowed   : 1;
 	struct list_head pv_target;
 	struct reloc *relocs;
 };
tools/objtool/include/objtool/special.h +1 -0
index 86d4af9c5aa9..ce4759358ec4 100644
@@ -20,6 +20,7 @@ struct special_alt {
 	bool skip_alt;
 	bool jump_or_nop;
 	u8 key_addend;
+	struct symbol *key;
 
 	struct section *orig_sec;
 	unsigned long orig_off;
tools/objtool/special.c +15 -3
index 097a69db82a0..982d5cb55e1b 100644
@@ -119,14 +119,26 @@ static int get_alt_entry(struct elf *elf, const struct special_entry *entry,
 
 	if (entry->key) {
 		struct reloc *key_reloc;
+		struct symbol *key;
+		s64 key_addend;
 
 		key_reloc = find_reloc_by_dest(elf, sec, offset + entry->key);
 		if (!key_reloc) {
-			WARN_FUNC("can't find key reloc",
-				  sec, offset + entry->key);
+			WARN_FUNC("can't find key reloc", sec, offset + entry->key);
 			return -1;
 		}
-		alt->key_addend = reloc_addend(key_reloc);
+
+		key = key_reloc->sym;
+		key_addend = reloc_addend(key_reloc);
+
+		if (key->type == STT_SECTION)
+			key = find_symbol_by_offset(key->sec, key_addend & ~3);
+
+		/* embedded keys not supported */
+		if (key) {
+			alt->key = key;
+			alt->key_addend = key_addend;
+		}
 	}
 
 	return 0;
Signature · 1 line
2.47.0
#4 lore raw

Re: [PATCH v2 3/3] objtool: Add noinstr validation for static branches/calls

Quoted 10 lines · Josh Poimboeuf
tools/objtool/check.c +5 -0
@@ -3532,14 +3577,29 @@ static inline bool noinstr_call_dest(struct objtool_file *file,
      return false;
 }
 
+static char *static_call_name(struct symbol *func)
+{
+	return func->name + strlen("__SCT__");
+}
+
 static int validate_call(struct objtool_file *file,

tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3591

Quoted 10 lines
tools/objtool/check.c +3 -4
                       struct instruction *insn,
                       struct insn_state *state)
 {
-	if (state->noinstr && state->instr <= 0 &&
-	    !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
-		WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
-		return 1;
+	if (state->noinstr && state->instr <= 0) {
+		struct symbol *dest = insn_call_dest(insn);
+

Interestingly only after I tried making __flush_tlb_all() noinstr did this start causing a segfault due to a NULL dest. I added a NULL check here but I wonder if I'm not just plastering over some other issue?

Quoted 14 lines
tools/objtool/check.c +11 -0
+		if (dest->static_call_tramp) {
+			if (!dest->noinstr_allowed) {
+				WARN_INSN(insn, "%s: non-RO static call usage in noinstr",
+					  static_call_name(dest));
+			}
+
+		} else if (!noinstr_call_allowed(file, insn, dest)) {
+			WARN_INSN(insn, "call to %s() leaves .noinstr.text section",
+				  call_dest_name(insn));
+			return 1;
+		}
      }
 
      if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
#5 lore raw

Re: [PATCH v2 3/3] objtool: Add noinstr validation for static branches/calls

Quoted 26 lines · Valentin Schneider
Quoted 10 lines · Josh Poimboeuf
tools/objtool/check.c +5 -0
@@ -3532,14 +3577,29 @@ static inline bool noinstr_call_dest(struct objtool_file *file,
      return false;
 }
 
+static char *static_call_name(struct symbol *func)
+{
+	return func->name + strlen("__SCT__");
+}
+
 static int validate_call(struct objtool_file *file,

tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3591

Quoted 10 lines
tools/objtool/check.c +3 -4
                       struct instruction *insn,
                       struct insn_state *state)
 {
-	if (state->noinstr && state->instr <= 0 &&
-	    !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
-		WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
-		return 1;
+	if (state->noinstr && state->instr <= 0) {
+		struct symbol *dest = insn_call_dest(insn);
+

Interestingly only after I tried making __flush_tlb_all() noinstr did this start causing a segfault due to a NULL dest. I added a NULL check here but I wonder if I'm not just plastering over some other issue?

Quoted 1 line

tools/objtool/check.c static inline bool noinstr_call_dest(struct objtool_file *file, · line 3592

Quoted 1 line
tools/objtool/check.c +1 -0
+		if (dest->static_call_tramp) {

Yeah, this line can probably just be:

		if (dest && dest->static_call_tramp)
Signature · 1 line
Josh
#6 lore raw
In reply to Josh Poimboeuf
Quoted 12 lines · Josh Poimboeuf

v2:
- Add some fixes reported by kbuild.

FWIW, this reports the following static keys:

- sched_clock_running
- __sched_clock_stable
- mds_idle_clear
- vmx_l1d_flush_cond
- stack_erasing_bypass

Quoted 5 lines

and the following static calls:

- pv_sched_clock
- x86_idle

And you event did the static calls, sweet! Thanks a ton Josh, lemme catch up with emails and then I'll look into these.

author maintainer
#7 lore raw
In reply to Josh Poimboeuf
Quoted 11 lines · Josh Poimboeuf

v2:
- Add some fixes reported by kbuild.

FWIW, this reports the following static keys:

- sched_clock_running
- __sched_clock_stable
- mds_idle_clear
- vmx_l1d_flush_cond
- stack_erasing_bypass

- perf_lopwr_cb

Quoted 4 lines

and the following static calls:

- pv_sched_clock
- x86_idle

Signature · 1 line
Josh
#8 lore raw
Quoted 5 lines · Josh Poimboeuf
Quoted 4 lines · Josh Poimboeuf

v2:
- Add some fixes reported by kbuild.

Quoted 8 lines
Quoted 7 lines

FWIW, this reports the following static keys:

- sched_clock_running
- __sched_clock_stable
- mds_idle_clear
- vmx_l1d_flush_cond
- stack_erasing_bypass

sched_clock_running, __sched_clock_stable and mds_idle_clear are already covered in v3

vmx_l1d_flush_cond is modified
- at init via __init vmx_init()
- via a module parameter write ending in vmentry_l1d_flush_set() So either init or user/admin action, IMO that's a "don't touch it" scenario.

stack_erasing_bypass is modified only via a sysctl write, so IMO same argument.

Quoted 5 lines
Quoted 4 lines

and the following static calls:

- pv_sched_clock
- x86_idle

- perf_lopwr_cb

All of these are only ever modified at __init, so they can be made __ro_after_init.

Quoted 3 lines
Signature · 2 lines
-- 
Josh