Skip to contentSkip to conversation

Add static_call()

PATCH RESEND Superseded Peter Zijlstra 13 participants 51 messages Mar 24, 2020 – Apr 6, 2020

Subsystems (16)
x86 BPF perf events Tracing +12 more subsystems

v3 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

v3 cover Add static_call()

[resend; because of the typo in the lkml address]

static_call(), is the idea of static_branch() applied to indirect function calls. Remove a data load (indirection) by modifying the text.

The inline implementation still relies on objtool to generate the .static_call_sites section, mostly because this is a natural place for x86_64 and works for both GCC and LLVM. Other architectures can pick other means if/when they implement the inline patching. The out-of-line (aka. trampoline) variant doesn't require this.

Patches can also be found here:

git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/static_call

Comments?

author maintainer
#1 lore raw

PATCH RESEND v3 1/17 notifier: Fix broken error handling pattern

The current notifiers have the following error handling pattern all over the place:

	int err, nr;

	err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr);
	if (err & NOTIFIER_STOP_MASK)
		__foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL)

And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless.

Fix this by replacing all the __foo_notifier_call_chain() functions with foo_notifier_call_chain_robust() that embeds the above pattern, but ensures it is inside a single lock region.

Note: I switched atomic_notifier_call_chain_robust() to use
the spinlock, since RCU cannot provide the guarantee required for the recovery.

Note: software_resume() error handling was broken afaict.

---
 include/linux/notifier.h           |   15 +--
 kernel/cpu_pm.c                    |   48 ++++--------
 kernel/notifier.c                  |  144 ++++++++++++++++++++++---------------
 kernel/power/hibernate.c           |   26 +++---
 kernel/power/main.c                |    8 +-
 kernel/power/power.h               |    3 
 kernel/power/suspend.c             |   14 +--
 kernel/power/user.c                |   14 +--
 tools/power/pm-graph/sleepgraph.py |    2 
 9 files changed, 141 insertions(+), 133 deletions(-)
include/linux/notifier.h +7 -8
@@ -161,20 +161,19 @@ extern int srcu_notifier_chain_unregiste
 
 extern int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
 		unsigned long val, void *v);
-extern int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
-	unsigned long val, void *v, int nr_to_call, int *nr_calls);
 extern int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
 		unsigned long val, void *v);
-extern int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
-	unsigned long val, void *v, int nr_to_call, int *nr_calls);
 extern int raw_notifier_call_chain(struct raw_notifier_head *nh,
 		unsigned long val, void *v);
-extern int __raw_notifier_call_chain(struct raw_notifier_head *nh,
-	unsigned long val, void *v, int nr_to_call, int *nr_calls);
 extern int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
 		unsigned long val, void *v);
-extern int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
-	unsigned long val, void *v, int nr_to_call, int *nr_calls);
+
+extern int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
+		unsigned long val_up, unsigned long val_down, void *v);
+extern int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
+		unsigned long val_up, unsigned long val_down, void *v);
+extern int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
+		unsigned long val_up, unsigned long val_down, void *v);
 
 #define NOTIFY_DONE		0x0000		/* Don't care */
 #define NOTIFY_OK		0x0001		/* Suits me */
kernel/cpu_pm.c +18 -30

86 lines not on this page · raw message

kernel/notifier.c +88 -56

240 lines not on this page · raw message

kernel/power/hibernate.c +12 -14
@@ -694,8 +694,8 @@ static int load_image_and_restore(void)
  */
 int hibernate(void)
 {
-	int error, nr_calls = 0;
 	bool snapshot_test = false;
+	int error;
 
 	if (!hibernation_available()) {
 		pm_pr_dbg("Hibernation not available.\n");
@@ -711,11 +711,9 @@ int hibernate(void)
 
 	pr_info("hibernation entry\n");
 	pm_prepare_console();
-	error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
-	if (error) {
-		nr_calls--;
-		goto Exit;
-	}
+	error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
+	if (error)
+		goto Restore;
 
 	ksys_sync_helper();
 
@@ -773,7 +771,8 @@ int hibernate(void)
 	/* Don't bother checking whether freezer_test_done is true */
 	freezer_test_done = false;
  Exit:
-	__pm_notifier_call_chain(PM_POST_HIBERNATION, nr_calls, NULL);
+	pm_notifier_call_chain(PM_POST_HIBERNATION);
+ Restore:
 	pm_restore_console();
 	atomic_inc(&snapshot_device_available);
  Unlock:
@@ -801,7 +800,7 @@ int hibernate(void)
  */
 static int software_resume(void)
 {
-	int error, nr_calls = 0;
+	int error;
 
 	/*
 	 * If the user said "noresume".. bail out early.
@@ -888,11 +887,9 @@ static int software_resume(void)
 
 	pr_info("resume from hibernation\n");
 	pm_prepare_console();
-	error = __pm_notifier_call_chain(PM_RESTORE_PREPARE, -1, &nr_calls);
-	if (error) {
-		nr_calls--;
-		goto Close_Finish;
-	}
+	error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
+	if (error)
+		goto Restore;
 
 	pm_pr_dbg("Preparing processes for hibernation restore.\n");
 	error = freeze_processes();
@@ -901,7 +898,8 @@ static int software_resume(void)
 	error = load_image_and_restore();
 	thaw_processes();
  Finish:
-	__pm_notifier_call_chain(PM_POST_RESTORE, nr_calls, NULL);
+	pm_notifier_call_chain(PM_POST_RESTORE);
+ Restore:
 	pm_restore_console();
 	pr_info("resume failed (%d)\n", error);
 	atomic_inc(&snapshot_device_available);
kernel/power/main.c +4 -4
@@ -80,18 +80,18 @@ int unregister_pm_notifier(struct notifi
 }
 EXPORT_SYMBOL_GPL(unregister_pm_notifier);
 
-int __pm_notifier_call_chain(unsigned long val, int nr_to_call, int *nr_calls)
+int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down)
 {
 	int ret;
 
-	ret = __blocking_notifier_call_chain(&pm_chain_head, val, NULL,
-						nr_to_call, nr_calls);
+	ret = blocking_notifier_call_chain_robust(&pm_chain_head, val_up, val_down, NULL);
 
 	return notifier_to_errno(ret);
 }
+
 int pm_notifier_call_chain(unsigned long val)
 {
-	return __pm_notifier_call_chain(val, -1, NULL);
+	return blocking_notifier_call_chain(&pm_chain_head, val, NULL);
 }
 
 /* If set, devices may be suspended and resumed asynchronously. */
kernel/power/power.h +1 -2
@@ -210,8 +210,7 @@ static inline void suspend_test_finish(c
 
 #ifdef CONFIG_PM_SLEEP
 /* kernel/power/main.c */
-extern int __pm_notifier_call_chain(unsigned long val, int nr_to_call,
-				    int *nr_calls);
+extern int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down);
 extern int pm_notifier_call_chain(unsigned long val);
 #endif
 
kernel/power/suspend.c +6 -8
@@ -342,18 +342,16 @@ static int suspend_test(int level)
  */
 static int suspend_prepare(suspend_state_t state)
 {
-	int error, nr_calls = 0;
+	int error;
 
 	if (!sleep_state_supported(state))
 		return -EPERM;
 
 	pm_prepare_console();
 
-	error = __pm_notifier_call_chain(PM_SUSPEND_PREPARE, -1, &nr_calls);
-	if (error) {
-		nr_calls--;
-		goto Finish;
-	}
+	error = pm_notifier_call_chain_robust(PM_SUSPEND_PREPARE, PM_POST_SUSPEND);
+	if (error)
+		goto Restore;
 
 	trace_suspend_resume(TPS("freeze_processes"), 0, true);
 	error = suspend_freeze_processes();
@@ -363,8 +361,8 @@ static int suspend_prepare(suspend_state
 
 	suspend_stats.failed_freeze++;
 	dpm_save_failed_step(SUSPEND_FREEZE);
- Finish:
-	__pm_notifier_call_chain(PM_POST_SUSPEND, nr_calls, NULL);
+	pm_notifier_call_chain(PM_POST_SUSPEND);
+ Restore:
 	pm_restore_console();
 	return error;
 }
kernel/power/user.c +4 -10
@@ -44,7 +44,7 @@ atomic_t snapshot_device_available = ATO
 static int snapshot_open(struct inode *inode, struct file *filp)
 {
 	struct snapshot_data *data;
-	int error, nr_calls = 0;
+	int error;
 
 	if (!hibernation_available())
 		return -EPERM;
@@ -71,9 +71,7 @@ static int snapshot_open(struct inode *i
 			swap_type_of(swsusp_resume_device, 0, NULL) : -1;
 		data->mode = O_RDONLY;
 		data->free_bitmaps = false;
-		error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
-		if (error)
-			__pm_notifier_call_chain(PM_POST_HIBERNATION, --nr_calls, NULL);
+		error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
 	} else {
 		/*
 		 * Resuming.  We may need to wait for the image device to
@@ -83,15 +81,11 @@ static int snapshot_open(struct inode *i
 
 		data->swap = -1;
 		data->mode = O_WRONLY;
-		error = __pm_notifier_call_chain(PM_RESTORE_PREPARE, -1, &nr_calls);
+		error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
 		if (!error) {
 			error = create_basic_memory_bitmaps();
 			data->free_bitmaps = !error;
-		} else
-			nr_calls--;
-
-		if (error)
-			__pm_notifier_call_chain(PM_POST_RESTORE, nr_calls, NULL);
+		}
 	}
 	if (error)
 		atomic_inc(&snapshot_device_available);
tools/power/pm-graph/sleepgraph.py +1 -1
@@ -167,7 +167,7 @@ import base64
 	tracefuncs = {
 		'sys_sync': {},
 		'ksys_sync': {},
-		'__pm_notifier_call_chain': {},
+		'pm_notifier_call_chain_robust': {},
 		'pm_prepare_console': {},
 		'pm_notifier_call_chain': {},
 		'freeze_processes': {},
author maintainer
#2 lore raw

PATCH RESEND v3 2/17 module: Fix up module_notifier return values

In reply to Peter Zijlstra

While auditing all module notifiers I noticed a whole bunch of fail wrt the return value. Notifiers have a 'special' return semantics.

As is; NOTIFY_DONE vs NOTIFY_OK is a bit vague; but notifier_from_errno(0) results in NOTIFY_OK and NOTIFY_DONE has a comment that says "Don't care".

Quoted 1 line

From this I've used NOTIFY_DONE when the function completely ignores

the callback and notifier_to_error() isn't used.

---
 drivers/oprofile/buffer_sync.c |    4 ++--
 kernel/trace/bpf_trace.c       |    8 ++++++--
 kernel/trace/trace.c           |    2 +-
 kernel/trace/trace_events.c    |    2 +-
 kernel/trace/trace_printk.c    |    4 ++--
 kernel/tracepoint.c            |    2 +-
 6 files changed, 13 insertions(+), 9 deletions(-)
drivers/oprofile/buffer_sync.c +2 -2
@@ -116,7 +116,7 @@ module_load_notify(struct notifier_block
 {
 #ifdef CONFIG_MODULES
 	if (val != MODULE_STATE_COMING)
-		return 0;
+		return NOTIFY_DONE;
 
 	/* FIXME: should we process all CPU buffers ? */
 	mutex_lock(&buffer_mutex);
@@ -124,7 +124,7 @@ module_load_notify(struct notifier_block
 	add_event_entry(MODULE_LOADED_CODE);
 	mutex_unlock(&buffer_mutex);
 #endif
-	return 0;
+	return NOTIFY_OK;
 }
 
 
kernel/trace/bpf_trace.c +6 -2
@@ -1451,10 +1451,11 @@ static int bpf_event_notify(struct notif
 {
 	struct bpf_trace_module *btm, *tmp;
 	struct module *mod = module;
+	int ret = 0;
 
 	if (mod->num_bpf_raw_events == 0 ||
 	    (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
-		return 0;
+		goto out;
 
 	mutex_lock(&bpf_module_mutex);
 
@@ -1464,6 +1465,8 @@ static int bpf_event_notify(struct notif
 		if (btm) {
 			btm->module = module;
 			list_add(&btm->list, &bpf_trace_modules);
+		} else {
+			ret = -ENOMEM;
 		}
 		break;
 	case MODULE_STATE_GOING:
@@ -1479,7 +1482,8 @@ static int bpf_event_notify(struct notif
 
 	mutex_unlock(&bpf_module_mutex);
 
-	return 0;
+out:
+	return notifier_from_errno(ret);
 }
 
 static struct notifier_block bpf_module_nb = {
kernel/trace/trace.c +1 -1
@@ -8696,7 +8696,7 @@ static int trace_module_notify(struct no
 		break;
 	}
 
-	return 0;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block trace_module_nb = {
kernel/trace/trace_events.c +1 -1
@@ -2442,7 +2442,7 @@ static int trace_module_notify(struct no
 	mutex_unlock(&trace_types_lock);
 	mutex_unlock(&event_mutex);
 
-	return 0;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block trace_module_nb = {
kernel/trace/trace_printk.c +2 -2
@@ -95,7 +95,7 @@ static int module_trace_bprintk_format_n
 		if (val == MODULE_STATE_COMING)
 			hold_module_trace_bprintk_format(start, end);
 	}
-	return 0;
+	return NOTIFY_OK;
 }
 
 /*
@@ -173,7 +173,7 @@ __init static int
 module_trace_bprintk_format_notify(struct notifier_block *self,
 		unsigned long val, void *data)
 {
-	return 0;
+	return NOTIFY_OK;
 }
 static inline const char **
 find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
kernel/tracepoint.c +1 -1
@@ -521,7 +521,7 @@ static int tracepoint_module_notify(stru
 	case MODULE_STATE_UNFORMED:
 		break;
 	}
-	return ret;
+	return notifier_from_errno(ret);
 }
 
 static struct notifier_block tracepoint_module_nb = {
author maintainer
#3 lore raw

PATCH RESEND v3 3/17 module: Properly propagate MODULE_STATE_COMING failure

In reply to Peter Zijlstra

Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.

There were already MODULE_STATE_COMING notifiers that failed; notably:

- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()

By propagating this error, we fix those users.

---
 kernel/module.c |   10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +7 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(struct
 	if (err)
 		return err;
 
-	blocking_notifier_call_chain(&module_notify_list,
-				     MODULE_STATE_COMING, mod);
-	return 0;
+	err = blocking_notifier_call_chain_robust(&module_notify_list,
+			MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
+	err = notifier_to_errno(err);
+	if (err)
+		klp_module_going(mod);
+
+	return err;
 }
 
 static int unknown_module_param_cb(char *param, char *val, const char *modname,
Acked-by on v3 3/17
#4 lore raw

Re: [RESEND][PATCH v3 03/17] module: Properly propagate MODULE_STATE_COMING failure

+++ Peter Zijlstra [24/03/20 14:56 +0100]:

Quoted 27 lines

Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.

There were already MODULE_STATE_COMING notifiers that failed; notably:

- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()

By propagating this error, we fix those users.

---
kernel/module.c | 10 +++++++---

1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +0 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(struct
 	if (err)
 		return err;
 
-	blocking_notifier_call_chain(&module_notify_list,
-				     MODULE_STATE_COMING, mod);
-	return 0;

kernel/module.c static int prepare_coming_module(struct · line 3763

Quoted 10 lines
kernel/module.c +7 -0
+	err = blocking_notifier_call_chain_robust(&module_notify_list,
+			MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
+	err = notifier_to_errno(err);
+	if (err)
+		klp_module_going(mod);
+
+	return err;
 }
 
 static int unknown_module_param_cb(char *param, char *val, const char *modname,

This looks fine to me - klp_module_going() is only called after successful klp_module_coming(), and klp_module_going() is fine with mod->state still being MODULE_STATE_COMING here. Would be good to have livepatch folks double check. Which reminds me - Miroslav had pointed out in the past that if there is an error when calling the COMING notifiers, the GOING notifiers will be called while the mod->state is still MODULE_STATE_COMING. I've briefly looked through all the module notifiers and it looks like nobody is looking at mod->state directly at least.

#5 lore raw

Re: [RESEND][PATCH v3 03/17] module: Properly propagate MODULE_STATE_COMING failure

Quoted 49 lines · Jessica Yu

+++ Peter Zijlstra [24/03/20 14:56 +0100]:

Quoted 27 lines

Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.

There were already MODULE_STATE_COMING notifiers that failed; notably:

- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()

By propagating this error, we fix those users.

---
kernel/module.c | 10 +++++++---

1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +0 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(struct
 	if (err)
 		return err;
 
-	blocking_notifier_call_chain(&module_notify_list,
-				     MODULE_STATE_COMING, mod);
-	return 0;

kernel/module.c static int prepare_coming_module(struct · line 3763

Quoted 10 lines
kernel/module.c +7 -0
+	err = blocking_notifier_call_chain_robust(&module_notify_list,
+			MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
+	err = notifier_to_errno(err);
+	if (err)
+		klp_module_going(mod);
+
+	return err;
 }
 
 static int unknown_module_param_cb(char *param, char *val, const char *modname,

This looks fine to me - klp_module_going() is only called after successful klp_module_coming(), and klp_module_going() is fine with mod->state still being MODULE_STATE_COMING here. Would be good to have livepatch folks double check. Which reminds me - Miroslav had pointed out in the past that if there is an error when calling the COMING notifiers, the GOING notifiers will be called while the mod->state is still MODULE_STATE_COMING. I've briefly looked through all the module notifiers and it looks like nobody is looking at mod->state directly at least.

Quoted 1 line

Looks good to me. klp_module_going() is already called in other load_module() error scenarios so this should be fine from a livepatch standpoint.

Signature · 1 line
Josh
maintainer Reviewed-by
#6 lore raw

Re: [RESEND][PATCH v3 03/17] module: Properly propagate MODULE_STATE_COMING failure

In reply to Jessica Yu
Quoted 36 lines · Jessica Yu

+++ Peter Zijlstra [24/03/20 14:56 +0100]:

Quoted 35 lines

Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.

There were already MODULE_STATE_COMING notifiers that failed; notably:

- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()

By propagating this error, we fix those users.

---
kernel/module.c | 10 +++++++---

1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +7 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(struct
 if (err)
  return err;
 
-	blocking_notifier_call_chain(&module_notify_list,
-				     MODULE_STATE_COMING, mod);
-	return 0;
+	err = blocking_notifier_call_chain_robust(&module_notify_list,
+			MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
+	err = notifier_to_errno(err);
+	if (err)
+		klp_module_going(mod);
+
+	return err;
 }
 
Quoted 8 lines
Quoted 3 lines

static int unknown_module_param_cb(char *param, char *val, const char *modname,

This looks fine to me - klp_module_going() is only called after successful klp_module_coming(), and klp_module_going() is fine with mod->state still being MODULE_STATE_COMING here. Would be good to have livepatch folks double check.

Yes, it is ok.

Quoted 6 lines

Which reminds me - Miroslav had pointed
out in the past that if there is an error when calling the COMING notifiers, the GOING notifiers will be called while the mod->state is still MODULE_STATE_COMING. I've briefly looked through all the module notifiers and it looks like nobody is looking at mod->state directly at least.

Thanks for double-checking. I triple-checked and yes, it should be fine. All module notifiers check the value from the function parameter and not mod->state directly.

M

author maintainer
#7 lore raw

PATCH RESEND v3 4/17 jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved

In reply to Peter Zijlstra

Nothing ensures the module exists while we're iterating mod->jump_entries in __jump_label_mod_text_reserved(), take a module reference to ensure the module sticks around.

---
 kernel/jump_label.c |   10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/jump_label.c +8 -2
@@ -539,19 +539,25 @@ static void static_key_set_mod(struct st
 static int __jump_label_mod_text_reserved(void *start, void *end)
 {
 	struct module *mod;
+	int ret;
 
 	preempt_disable();
 	mod = __module_text_address((unsigned long)start);
 	WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
+	if (!try_module_get(mod))
+		mod = NULL;
 	preempt_enable();
 
 	if (!mod)
 		return 0;
 
-
-	return __jump_label_text_reserved(mod->jump_entries,
+	ret = __jump_label_text_reserved(mod->jump_entries,
 				mod->jump_entries + mod->num_jump_entries,
 				start, end);
+
+	module_put(mod);
+
+	return ret;
 }
 
 static void __jump_label_mod_update(struct static_key *key)
author maintainer
#8 lore raw

PATCH RESEND v3 5/17 compiler.h: Make __ADDRESSABLE() symbol truly unique

In reply to Peter Zijlstra

The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.

Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.

---
 include/linux/compiler.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -299,7 +299,7 @@ unsigned long read_word_at_a_time(const
  */
 #define __ADDRESSABLE(sym) \
 	static void * __section(.discard.addressable) __used \
-		__PASTE(__addressable_##sym, __LINE__) = (void *)&sym;
+		__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
 
 /**
  * offset_to_ptr - convert a relative memory offset to an absolute pointer
author maintainer
#9 lore raw

PATCH RESEND v3 6/17 static_call: Add basic static call infrastructure

In reply to Peter Zijlstra

Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.

The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:

https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]

There are two implementations, depending on arch support:

1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers

For more details, see the comments in include/linux/static_call.h.

[peterz: simplified interface]
---
 arch/Kconfig                      |    3 
 include/linux/static_call.h       |  137 ++++++++++++++++++++++++++++++++++++++
 include/linux/static_call_types.h |   15 ++++
 3 files changed, 155 insertions(+)
 create mode 100644 include/linux/static_call.h
 create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -941,6 +941,9 @@ config HAVE_SPARSE_SYSCALL_NR
 	  entries at 4000, 5000 and 6000 locations. This option turns on syscall
 	  related optimizations for a given architecture.
 
+config HAVE_STATIC_CALL
+	bool
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +137 -0

137 lines not on this page · raw message

include/linux/static_call_types.h new file +15 -0
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/stringify.h>
+
+#define STATIC_CALL_PREFIX	__SC__
+#define STATIC_CALL_PREFIX_STR	__stringify(STATIC_CALL_PREFIX)
+
+#define STATIC_CALL_NAME(name)	__PASTE(STATIC_CALL_PREFIX, name)
+
+#define STATIC_CALL_TRAMP(name)	    STATIC_CALL_NAME(name##_tramp)
+#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
+
+#endif /* _STATIC_CALL_TYPES_H */
#10 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 68 lines · Peter Zijlstra

Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.

The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:

https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181005081333.15018-1-ard.biesheuvel%40linaro.org&amp;data=02%7C01%7Cnamit%40vmware.com%7C48cb49e3bd65479a440008d7cfff207d%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C637206567005697724&amp;sdata=jLAdeYFX8C67QFeYtiguq9BfeN8zxjSa8gmtY%2F2nJQ8%3D&amp;reserved=0
https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181006015110.653946300%40goodmis.org&amp;data=02%7C01%7Cnamit%40vmware.com%7C48cb49e3bd65479a440008d7cfff207d%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C637206567005697724&amp;sdata=KBVx7EJo4EgzKaCccAaMP7C4L%2BYlQ6liO4wO7vv27MU%3D&amp;reserved=0

There are two implementations, depending on arch support:

1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers

For more details, see the comments in include/linux/static_call.h.

[peterz: simplified interface]

---

arch/Kconfig                      |    3 
include/linux/static_call.h       |  137 ++++++++++++++++++++++++++++++++++++++
include/linux/static_call_types.h |   15 ++++
3 files changed, 155 insertions(+)

create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h

arch/Kconfig +3 -0
@@ -941,6 +941,9 @@ config HAVE_SPARSE_SYSCALL_NR
 	  entries at 4000, 5000 and 6000 locations. This option turns on syscall
 	  related optimizations for a given architecture.
 
+config HAVE_STATIC_CALL
+	bool
+

source "kernel/gcov/Kconfig"

source "scripts/gcc-plugins/Kconfig"

include/linux/static_call.h new file +19 -0
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_STATIC_CALL_H
+#define _LINUX_STATIC_CALL_H
+
+/*
+ * Static call support
+ *
+ * Static calls use code patching to hard-code function pointers into direct
+ * branch instructions.  They give the flexibility of function pointers, but
+ * with improved performance.  This is especially important for cases where
+ * retpolines would otherwise be used, as retpolines can significantly impact
+ * performance.
+ *
+ *
+ * API overview:
+ *
+ *   DECLARE_STATIC_CALL(name, func);
+ *   DEFINE_STATIC_CALL(name, func);
+ *   static_call(name)(args...);

include/linux/static_call.h line 29

Quoted 10 lines
include/linux/static_call.h new file +10 -0
+ *   static_call_update(name, func);
+ *
+ * Usage example:
+ *
+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
+ *
+ *   # Define a 'my_name' reference, associated with func_a() by default
+ *   DEFINE_STATIC_CALL(my_name, func_a);

Do you want to support optional function attributes, such as “pure” and “const”?

author maintainer
#11 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 18 lines · Nadav Amit
Quoted 14 lines · Peter Zijlstra

+ * API overview:
+ *

+ *   DECLARE_STATIC_CALL(name, func);
+ *   DEFINE_STATIC_CALL(name, func);
+ *   static_call(name)(args...);
+ *   static_call_update(name, func);
+ *
+ * Usage example:
+ *
+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
Quoted 3 lines

+ *
+ * # Define a 'my_name' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_name, func_a);

Quoted 2 lines

Do you want to support optional function attributes, such as “pure” and “const”?

Do you see a need for that? And what is the syntax for a pointer to a pure function?

#12 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 21 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines · Peter Zijlstra

+ * API overview:
+ *

+ *   DECLARE_STATIC_CALL(name, func);
+ *   DEFINE_STATIC_CALL(name, func);
+ *   static_call(name)(args...);
+ *   static_call_update(name, func);
+ *
+ * Usage example:
+ *
+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);

+ *
+ * # Define a 'my_name' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_name, func_a);

Quoted 5 lines
Quoted 2 lines

Do you want to support optional function attributes, such as “pure” and “const”?

Do you see a need for that? And what is the syntax for a pointer to a pure function?

I think that the kernel underutilizes the pure attribute in general. Building it with "-Wsuggest-attribute=pure” results in many warnings. Function pointers such kvm_x86_ops.get_XXX() could have been candidates to use the “pure” attribute.

The syntax is what you would expect:

  static void __attribute__((pure))(*ptr)(void);

However, you have a point, gcc does not appear to respect “pure” for function pointers and emits a warning it is ignored. GCC apparently only respects “const”. In contrast clang appears to respect the pure attribute for function pointers.

author maintainer
#13 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 9 lines · Nadav Amit

I think that the kernel underutilizes the pure attribute in general. Building it with "-Wsuggest-attribute=pure” results in many warnings. Function pointers such kvm_x86_ops.get_XXX() could have been candidates to use the “pure” attribute.

The syntax is what you would expect:

  static void __attribute__((pure))(*ptr)(void);

Well, I didn't in fact expect that, because an attribute is not a type qualifier.

Quoted 4 lines

However, you have a point, gcc does not appear to respect “pure” for function pointers and emits a warning it is ignored. GCC apparently only respects “const”. In contrast clang appears to respect the pure attribute for function pointers.

Still, we can probably make it happen for static_call(), since it is a direct call to the trampoline, all we need to do is make sure the trampoline is declared pure.

It does however mean that static_call() inherits all the dangers and pit-falls of function pointers with some extra on top. It will be impossible to validate this stuff.

That is, you can static_call_update() with a pointer to a !pure function and you get to keep the pieces.

#14 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Nadav Amit

I think that the kernel underutilizes the pure attribute in general. Building it with "-Wsuggest-attribute=pure” results in many warnings. Function pointers such kvm_x86_ops.get_XXX() could have been candidates to use the “pure” attribute.

Quoted 6 lines
Quoted 3 lines

The syntax is what you would expect:

static void __attribute__((pure))(*ptr)(void);

Well, I didn't in fact expect that, because an attribute is not a type qualifier.

Just a small correction for my stupid example - pure function should always return a value.

Quoted 9 lines
Quoted 4 lines

However, you have a point, gcc does not appear to respect “pure” for function pointers and emits a warning it is ignored. GCC apparently only respects “const”. In contrast clang appears to respect the pure attribute for function pointers.

Still, we can probably make it happen for static_call(), since it is a direct call to the trampoline, all we need to do is make sure the trampoline is declared pure.

Quoted 6 lines

It does however mean that static_call() inherits all the dangers and pit-falls of function pointers with some extra on top. It will be impossible to validate this stuff.

That is, you can static_call_update() with a pointer to a !pure function and you get to keep the pieces.

I understand. Well, perhaps it can be added later, as anyhow GCC does not support it.

On another note - it may be beneficial to see if the infrastructure that you built can accommodate notifier-chains. It is not the most painful point, but it would be nice to deal with those as well. Since many of those are changed asynchronously, I am not sure it is the easiest thing to do.

author maintainer
#15 lore raw

Re: [RESEND][PATCH v3 06/17] static_call: Add basic static call infrastructure

Quoted 4 lines · Nadav Amit

On another note - it may be beneficial to see if the infrastructure that you built can accommodate notifier-chains. It is not the most painful point, but it would be nice to deal with those as well. Since many of those are changed asynchronously, I am not sure it is the easiest thing to do.

You mean, like patch 12 does?

author maintainer
#16 lore raw

PATCH RESEND v3 7/17 static_call: Add inline static call infrastructure

In reply to Peter Zijlstra

Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.

Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:

https://lkml.kernel.org/r/[email protected]

This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.

For more details, see the comments in include/linux/static_call.h.

[peterz: simplified interface; merged trampolines]
---
 arch/Kconfig                      |    4 
 include/asm-generic/vmlinux.lds.h |    7 
 include/linux/module.h            |    5 
 include/linux/static_call.h       |   37 ++++
 include/linux/static_call_types.h |   10 +
 kernel/Makefile                   |    1 
 kernel/module.c                   |    5 
 kernel/static_call.c              |  302 ++++++++++++++++++++++++++++++++++++++
 8 files changed, 370 insertions(+), 1 deletion(-)
 create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -963,6 +963,10 @@ config ARCH_HAS_MEM_ENCRYPT
 config HAVE_STATIC_CALL
 	bool
 
+config HAVE_STATIC_CALL_INLINE
+	bool
+	depends on HAVE_STATIC_CALL
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -359,6 +359,12 @@
 	KEEP(*(__jump_table))						\
 	__stop___jump_table = .;
 
+#define STATIC_CALL_DATA						\
+	. = ALIGN(8);							\
+	__start_static_call_sites = .;					\
+	KEEP(*(.static_call_sites))					\
+	__stop_static_call_sites = .;
+
 /*
  * Allow architectures to handle ro_after_init data on their
  * own by defining an empty RO_AFTER_INIT_DATA.
@@ -368,6 +374,7 @@
 	__start_ro_after_init = .;					\
 	*(.data..ro_after_init)						\
 	JUMP_TABLE_DATA							\
+	STATIC_CALL_DATA						\
 	__end_ro_after_init = .;
 #endif
 
include/linux/module.h +5 -0
@@ -22,6 +22,7 @@
 #include <linux/error-injection.h>
 #include <linux/tracepoint-defs.h>
 #include <linux/srcu.h>
+#include <linux/static_call_types.h>
 
 #include <linux/percpu.h>
 #include <asm/module.h>
@@ -476,6 +477,10 @@ struct module {
 	unsigned int num_ftrace_callsites;
 	unsigned long *ftrace_callsites;
 #endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	int num_static_call_sites;
+	struct static_call_site *static_call_sites;
+#endif
 
 #ifdef CONFIG_LIVEPATCH
 	bool klp; /* Is this a livepatch module? */
include/linux/static_call.h +36 -1
@@ -81,7 +81,42 @@ extern void arch_static_call_transform(v
 			     STATIC_CALL_TRAMP_ADDR(name), func);	\
 })
 
-#if defined(CONFIG_HAVE_STATIC_CALL)
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
+struct static_call_mod {
+	struct static_call_mod *next;
+	struct module *mod; /* for vmlinux, mod == NULL */
+	struct static_call_site *sites;
+};
+
+struct static_call_key {
+	void *func;
+	struct static_call_mod *next;
+};
+
+extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
+extern int static_call_mod_init(struct module *mod);
+
+#define DEFINE_STATIC_CALL(name, _func)					\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = _func,						\
+		.next = NULL,						\
+	};								\
+	__ADDRESSABLE(STATIC_CALL_NAME(name));				\
+	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
+#define static_call(name)	STATIC_CALL_TRAMP(name)
+
+#define EXPORT_STATIC_CALL(name)					\
+	EXPORT_SYMBOL(STATIC_CALL_NAME(name));				\
+	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+
+#define EXPORT_STATIC_CALL_GPL(name)					\
+	EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name));			\
+	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+
+#elif defined(CONFIG_HAVE_STATIC_CALL)
 
 struct static_call_key {
 	void *func;
include/linux/static_call_types.h +10 -0
@@ -2,6 +2,7 @@
 #ifndef _STATIC_CALL_TYPES_H
 #define _STATIC_CALL_TYPES_H
 
+#include <linux/types.h>
 #include <linux/stringify.h>
 
 #define STATIC_CALL_PREFIX	__SC__
@@ -12,4 +13,13 @@
 #define STATIC_CALL_TRAMP(name)	    STATIC_CALL_NAME(name##_tramp)
 #define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
 
+/*
+ * The static call site table needs to be created by external tooling (objtool
+ * or a compiler plugin).
+ */
+struct static_call_site {
+	s32 addr;
+	s32 key;
+};
+
 #endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -107,6 +107,7 @@ obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
 obj-$(CONFIG_KCSAN) += kcsan/
+obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/module.c +5 -0
@@ -3188,6 +3188,11 @@ static int find_module_sections(struct m
 					    sizeof(*mod->ei_funcs),
 					    &mod->num_ei_funcs);
 #endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	mod->static_call_sites = section_objs(info, ".static_call_sites",
+					      sizeof(*mod->static_call_sites),
+					      &mod->num_static_call_sites);
+#endif
 	mod->extable = section_objs(info, "__ex_table",
 				    sizeof(*mod->extable), &mod->num_exentries);
 
kernel/static_call.c new file +302 -0

302 lines not on this page · raw message

maintainer
#17 lore raw

Re: [RESEND][PATCH v3 07/17] static_call: Add inline static call infrastructure

Quoted 4 lines · Peter Zijlstra
include/linux/static_call.h +4 -0
+#define DEFINE_STATIC_CALL(name, _func)					\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = _func,						\

include/linux/static_call.h extern void arch_static_call_transform(v · line 113

Quoted 10 lines
include/linux/static_call.h +10 -0
+		.next = NULL,						\
+	};								\
+	__ADDRESSABLE(STATIC_CALL_NAME(name));				\
+	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
+#define static_call(name)	STATIC_CALL_TRAMP(name)
+
+#define EXPORT_STATIC_CALL(name)					\
+	EXPORT_SYMBOL(STATIC_CALL_NAME(name));				\
+	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))

I think we want only the _GPL versions below - not those. As you said on IRC, jump_label/static_branch is GPL only.

Quoted 4 lines
include/linux/static_call.h +4 -0
+
+#define EXPORT_STATIC_CALL_GPL(name)					\
+	EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name));			\
+	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
author maintainer
#18 lore raw

PATCH RESEND v3 8/17 static_call: Avoid kprobes on inline static_call()s

In reply to Peter Zijlstra

Similar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.

---
 arch/x86/kernel/kprobes/opt.c |    4 +-
 include/linux/static_call.h   |   11 +++++++
 kernel/kprobes.c              |    2 +
 kernel/static_call.c          |   64 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+), 1 deletion(-)
arch/x86/kernel/kprobes/opt.c +3 -1
@@ -16,6 +16,7 @@
 #include <linux/kallsyms.h>
 #include <linux/ftrace.h>
 #include <linux/frame.h>
+#include <linux/static_call.h>
 
 #include <asm/text-patching.h>
 #include <asm/cacheflush.h>
@@ -188,7 +189,8 @@ static int copy_optimized_instructions(u
 	/* Check whether the address range is reserved */
 	if (ftrace_text_reserved(src, src + len - 1) ||
 	    alternatives_text_reserved(src, src + len - 1) ||
-	    jump_label_text_reserved(src, src + len - 1))
+	    jump_label_text_reserved(src, src + len - 1) ||
+	    static_call_text_reserved(src, src + len - 1))
 		return -EBUSY;
 
 	return len;
include/linux/static_call.h +11 -0
@@ -93,6 +93,7 @@ struct static_call_key {
 
 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
 extern int static_call_mod_init(struct module *mod);
+extern int static_call_text_reserved(void *start, void *end);
 
 #define DEFINE_STATIC_CALL(name, _func)					\
 	DECLARE_STATIC_CALL(name, _func);				\
@@ -137,6 +138,11 @@ void __static_call_update(struct static_
 	cpus_read_unlock();
 }
 
+static inline int static_call_text_reserved(void *start, void *end)
+{
+	return 0;
+}
+
 #define EXPORT_STATIC_CALL(name)	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 #define EXPORT_STATIC_CALL_GPL(name)	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 
@@ -161,6 +167,11 @@ void __static_call_update(struct static_
 	WRITE_ONCE(key->func, func);
 }
 
+static inline int static_call_text_reserved(void *start, void *end)
+{
+	return 0;
+}
+
 #define EXPORT_STATIC_CALL(name)	EXPORT_SYMBOL(STATIC_CALL_NAME(name))
 #define EXPORT_STATIC_CALL_GPL(name)	EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name))
 
kernel/kprobes.c +2 -0
@@ -35,6 +35,7 @@
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/jump_label.h>
+#include <linux/static_call.h>
 
 #include <asm/sections.h>
 #include <asm/cacheflush.h>
@@ -1539,6 +1540,7 @@ static int check_kprobe_address_safe(str
 	if (!kernel_text_address((unsigned long) p->addr) ||
 	    within_kprobe_blacklist((unsigned long) p->addr) ||
 	    jump_label_text_reserved(p->addr, p->addr) ||
+	    static_call_text_reserved(p->addr, p->addr) ||
 	    find_bug((unsigned long)p->addr)) {
 		ret = -EINVAL;
 		goto out;
kernel/static_call.c +64 -0

78 lines not on this page · raw message

author maintainer
#19 lore raw

PATCH RESEND v3 9/17 x86/static_call: Add out-of-line static call implementation

In reply to Peter Zijlstra

Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.

[peterz: fixed trampoline, rewrote patching code]
---
 arch/x86/Kconfig                   |    1 +
 arch/x86/include/asm/static_call.h |   22 ++++++++++++++++++++++
 arch/x86/kernel/Makefile           |    1 +
 arch/x86/kernel/static_call.c      |   31 +++++++++++++++++++++++++++++++
 4 files changed, 55 insertions(+)
 create mode 100644 arch/x86/include/asm/static_call.h
 create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -205,6 +205,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_STATIC_CALL
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +22 -0
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+#include <asm/text-patching.h>
+
+/*
+ * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
+ * does a direct jump to the function.  The direct jump gets patched by
+ * static_call_update().
+ */
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	jmp.d32 " #func "				\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-y			+= tsc.o tsc_msr.o io_delay.o rt
 obj-y			+= pci-iommu_table.o
 obj-y			+= resource.o
 obj-y			+= irqflags.o
+obj-y			+= static_call.o
 
 obj-y				+= process.o
 obj-y				+= fpu/
arch/x86/kernel/static_call.c new file +31 -0
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/static_call.h>
+#include <linux/memory.h>
+#include <linux/bug.h>
+#include <asm/text-patching.h>
+
+static void __static_call_transform(void *insn, u8 opcode, void *func)
+{
+	const void *code = text_gen_insn(opcode, (long)insn, (long)func);
+
+	if (WARN_ONCE(*(u8 *)insn != opcode,
+		      "unexpected static call insn opcode 0x%x at %pS\n",
+		      opcode, insn))
+		return;
+
+	if (memcmp(insn, code, CALL_INSN_SIZE) == 0)
+		return;
+
+	text_poke_bp(insn, code, CALL_INSN_SIZE, NULL);
+}
+
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	mutex_lock(&text_mutex);
+
+	if (tramp)
+		__static_call_transform(tramp, JMP32_INSN_OPCODE, func);
+
+	mutex_unlock(&text_mutex);
+}
+EXPORT_SYMBOL_GPL(arch_static_call_transform);
maintainer
#20 lore raw

Re: [RESEND][PATCH v3 09/17] x86/static_call: Add out-of-line static call implementation

arch/x86/kernel/static_call.c line 19

Quoted 13 lines · Peter Zijlstra
arch/x86/kernel/static_call.c new file +13 -0
+static void __static_call_transform(void *insn, u8 opcode, void *func)
+{
+	const void *code = text_gen_insn(opcode, (long)insn, (long)func);
+
+	if (WARN_ONCE(*(u8 *)insn != opcode,
+		      "unexpected static call insn opcode 0x%x at %pS\n",
+		      opcode, insn))
+		return;
+
+	if (memcmp(insn, code, CALL_INSN_SIZE) == 0)
+		return;
+
+	text_poke_bp(insn, code, CALL_INSN_SIZE, NULL);

Right, this is working with CALL_INSN_SIZE but ...

arch/x86/kernel/static_call.c line 27

Quoted 8 lines
arch/x86/kernel/static_call.c new file +8 -0
+}
+
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	mutex_lock(&text_mutex);
+
+	if (tramp)
+		__static_call_transform(tramp, JMP32_INSN_OPCODE, func);

... it gets called with JMP32_INSN_OPCODE too. I mean, both lengths are equal and all - it is just a bit confusing at a first glance. Maybe slap a small comment that it is ok.

Thx.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#21 lore raw

Re: [RESEND][PATCH v3 09/17] x86/static_call: Add out-of-line static call implementation

In reply to Peter Zijlstra
Quoted 42 lines · Peter Zijlstra

Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.

[peterz: fixed trampoline, rewrote patching code]

---

arch/x86/Kconfig                   |    1 +
arch/x86/include/asm/static_call.h |   22 ++++++++++++++++++++++
arch/x86/kernel/Makefile           |    1 +
arch/x86/kernel/static_call.c      |   31 +++++++++++++++++++++++++++++++
4 files changed, 55 insertions(+)

create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c

arch/x86/Kconfig +1 -0
@@ -205,6 +205,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_STATIC_CALL
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +12 -0
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+#include <asm/text-patching.h>
+
+/*
+ * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
+ * does a direct jump to the function.  The direct jump gets patched by
+ * static_call_update().
+ */
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\

arch/x86/include/asm/static_call.h line 22

Quoted 10 lines
arch/x86/include/asm/static_call.h new file +10 -0
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	jmp.d32 " #func "				\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */

Hi Peter,

Coming here from https://github.com/ClangBuiltLinux/linux/issues/974

jmp.d32 is not recognized by clang integrated assembler.
The syntax appears to be very rarely used. According to Debian Code Search, u-boot is the only project using this syntax.

In March 2017, gas added the pseudo prefix {disp32}
https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=86fa6981e7487e2c2df4337aa75ed2d93c32eaf2
which generalizes jmp.d32 ({disp32} jmp foo)

I wonder whether the instruction jmp.d32 can be replaced with the trick in arch/x86/include/asm/jump_label.h for clang portability.

% grep -A2 'jmp.d32' arch/x86/include/asm/jump_label.h

        /* Equivalent to "jmp.d32 \target" */
        .byte           0xe9
        .long           \target - .Lstatic_jump_after_\@

...

+ [email protected]

author maintainer
#22 lore raw

Re: [RESEND][PATCH v3 09/17] x86/static_call: Add out-of-line static call implementation

Quoted 16 lines · Fangrui Song

arch/x86/include/asm/static_call.h line 22

Quoted 11 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h new file +11 -0
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	jmp.d32 " #func "				\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */

Hi Peter,

Quoted 5 lines

Coming here from https://github.com/ClangBuiltLinux/linux/issues/974

jmp.d32 is not recognized by clang integrated assembler.
The syntax appears to be very rarely used. According to Debian Code Search, u-boot is the only project using this syntax.

*groan*... I was going to use it in more places :-/

Quoted 3 lines

In March 2017, gas added the pseudo prefix {disp32}
https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=86fa6981e7487e2c2df4337aa75ed2d93c32eaf2
which generalizes jmp.d32 ({disp32} jmp foo)

That's all well and cute, but I can't use that because its too new. Not until we raise the minimum gcc/bintils version to something that includes that.

Quoted 7 lines

I wonder whether the instruction jmp.d32 can be replaced with the trick in arch/x86/include/asm/jump_label.h for clang portability.

% grep -A2 'jmp.d32' arch/x86/include/asm/jump_label.h

        /* Equivalent to "jmp.d32 \target" */
        .byte           0xe9
        .long           \target - .Lstatic_jump_after_\@

Sure, but I was hoping to move away from that since all assemblers should now support jmp.d32. Except of course, you have to go ruin things.

The thing is, jmp.d32 reads so much nicer than the above crap.

Also, I still need a meta instruction like:

nopjmp $label

what works just like 'jmp' but instead emits either a nop2 or a nop5. I tried various hacks to get GAS to emit that, but no luck :/

The only up-side from that new syntax is that I suppose we can go write:

{disp8} push \vec

without gas shitting itself and emitting a 5 byte push just because.. Except of course we can't, for the same reason I can't go around and use:

{disp32} jmp

in the above code.

#23 lore raw

Re: [RESEND][PATCH v3 09/17] x86/static_call: Add out-of-line static call implementation

Quoted 26 lines · Peter Zijlstra
Quoted 16 lines · Fangrui Song

arch/x86/include/asm/static_call.h line 22

Quoted 11 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h new file +11 -0
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)                  \
+   asm(".pushsection .text, \"ax\"                         \n"     \
+       ".align 4                                           \n"     \
+       ".globl " STATIC_CALL_TRAMP_STR(name) "             \n"     \
+       STATIC_CALL_TRAMP_STR(name) ":                      \n"     \
+       "   jmp.d32 " #func "                               \n"     \
+       ".type " STATIC_CALL_TRAMP_STR(name) ", @function   \n"     \
+       ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+       ".popsection                                        \n")
+
+#endif /* _ASM_STATIC_CALL_H */

Hi Peter,

Quoted 5 lines

Coming here from https://github.com/ClangBuiltLinux/linux/issues/974

jmp.d32 is not recognized by clang integrated assembler.
The syntax appears to be very rarely used. According to Debian Code Search, u-boot is the only project using this syntax.

*groan*... I was going to use it in more places :-/

Quoted 7 lines
Quoted 3 lines

In March 2017, gas added the pseudo prefix {disp32}
https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=86fa6981e7487e2c2df4337aa75ed2d93c32eaf2
which generalizes jmp.d32 ({disp32} jmp foo)

That's all well and cute, but I can't use that because its too new. Not until we raise the minimum gcc/bintils version to something that includes that.

If it seems like it would be useful, let us know. If it doesn't have some ridiculous baggage or inconsistency or unspecified behavior, we're generally happy to do so. We have finite resources so knowing where to focus our attention helps us sort the never ending TODO list. It's not easy to predict which feature we'll need to drop everything to implement next, so any help there would be much appreciated.

Quoted 38 lines
Quoted 8 lines

I wonder whether the instruction jmp.d32 can be replaced with the trick in arch/x86/include/asm/jump_label.h for clang portability.

% grep -A2 'jmp.d32' arch/x86/include/asm/jump_label.h

        /* Equivalent to "jmp.d32 \target" */
        .byte           0xe9
        .long           \target - .Lstatic_jump_after_\@

Sure, but I was hoping to move away from that since all assemblers should now support jmp.d32. Except of course, you have to go ruin things.

The thing is, jmp.d32 reads so much nicer than the above crap.

Also, I still need a meta instruction like:

nopjmp $label

what works just like 'jmp' but instead emits either a nop2 or a nop5. I tried various hacks to get GAS to emit that, but no luck :/

The only up-side from that new syntax is that I suppose we can go write:

{disp8} push \vec

without gas shitting itself and emitting a 5 byte push just because.. Except of course we can't, for the same reason I can't go around and use:

{disp32} jmp

in the above code.

--
You received this message because you are subscribed to the Google Groups "Clang Built Linux" group.
To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/clang-built-linux/20200406110438.GJ20730%40hirez.programming.kicks-ass.net.

Signature · 2 lines
Thanks,
~Nick Desaulniers
author maintainer
#24 lore raw

PATCH RESEND v3 10/17 x86/static_call: Add inline static call implementation for x86-64

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the inline static call implementation for x86-64. The generated code is identical to the out-of-line case, except we move the trampoline into it's own section.

Then objtool uses the trampoline section to detect all the call sites, which it writes into the .static_call_sites section.

During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.

[peterz: merged trampolines, put trampoline in section]
---
 arch/x86/Kconfig                        |    3 
 arch/x86/include/asm/static_call.h      |   15 +++
 arch/x86/kernel/static_call.c           |    3 
 arch/x86/kernel/vmlinux.lds.S           |    1 
 include/asm-generic/vmlinux.lds.h       |    6 +
 tools/include/linux/static_call_types.h |   25 ++++++
 tools/objtool/check.c                   |  128 +++++++++++++++++++++++++++++++-
 tools/objtool/check.h                   |    2 
 tools/objtool/elf.h                     |    1 
 tools/objtool/sync-check.sh             |    1 
 10 files changed, 182 insertions(+), 3 deletions(-)
 create mode 100644 tools/objtool/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -210,6 +210,7 @@ config X86
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
 	select HAVE_STATIC_CALL
+	select HAVE_STATIC_CALL_INLINE		if HAVE_STACK_VALIDATION
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
@@ -225,6 +226,7 @@ config X86
 	select RTC_MC146818_LIB
 	select SPARSE_IRQ
 	select SRCU
+	select STACK_VALIDATION			if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	select USER_STACKTRACE_SUPPORT
@@ -450,7 +452,6 @@ config GOLDFISH
 config RETPOLINE
 	bool "Avoid speculative indirect branches in kernel"
 	default y
-	select STACK_VALIDATION if HAVE_STACK_VALIDATION
 	help
 	  Compile kernel with the retpoline compiler options to guard against
 	  kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +14 -1
@@ -5,12 +5,25 @@
 #include <asm/text-patching.h>
 
 /*
+ * For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which
+ * uses the current value of the key->func pointer to do an indirect jump to
+ * the function.  This trampoline is only used during boot, before the call
+ * sites get patched by static_call_update().  The name of this trampoline has
+ * a magical aspect: objtool uses it to find static call sites so it can create
+ * the .static_call_sites section.
+ *
  * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
  * does a direct jump to the function.  The direct jump gets patched by
  * static_call_update().
+ *
+ * Having the trampoline in a special section has two benefits:
+ *  - it makes it 'easy' for objtool to find all the call-sites;
+ *  - it forces GCC to emit a JMP.d32 when it does tail-call optimization on
+ *    the call; since you cannot compute the relative displacement across
+ *    sections.
  */
 #define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\
-	asm(".pushsection .text, \"ax\"				\n"	\
+	asm(".pushsection .static_call.text, \"ax\"		\n"	\
 	    ".align 4						\n"	\
 	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
 	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
arch/x86/kernel/static_call.c +3 -0
@@ -26,6 +26,9 @@ void arch_static_call_transform(void *si
 	if (tramp)
 		__static_call_transform(tramp, JMP32_INSN_OPCODE, func);
 
+	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site)
+		__static_call_transform(site, CALL_INSN_OPCODE, func);
+
 	mutex_unlock(&text_mutex);
 }
 EXPORT_SYMBOL_GPL(arch_static_call_transform);
arch/x86/kernel/vmlinux.lds.S +1 -0
@@ -136,6 +136,7 @@ SECTIONS
 		IRQENTRY_TEXT
 		ALIGN_ENTRY_TEXT_END
 		SOFTIRQENTRY_TEXT
+		STATIC_CALL_TEXT
 		*(.fixup)
 		*(.gnu.warning)
 
include/asm-generic/vmlinux.lds.h +6 -0
@@ -609,6 +609,12 @@
 		*(.softirqentry.text)					\
 		__softirqentry_text_end = .;
 
+#define STATIC_CALL_TEXT						\
+		ALIGN_FUNCTION();					\
+		__static_call_text_start = .;				\
+		*(.static_call.text)					\
+		__static_call_text_end = .;
+
 /* Section used for early init (in .S files) */
 #define HEAD_TEXT  KEEP(*(.head.text))
 
tools/include/linux/static_call_types.h new file +25 -0
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+
+#define STATIC_CALL_PREFIX	__SC__
+#define STATIC_CALL_PREFIX_STR	__stringify(STATIC_CALL_PREFIX)
+
+#define STATIC_CALL_NAME(name)	__PASTE(STATIC_CALL_PREFIX, name)
+
+#define STATIC_CALL_TRAMP(name)	    STATIC_CALL_NAME(name##_tramp)
+#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
+
+/*
+ * The static call site table needs to be created by external tooling (objtool
+ * or a compiler plugin).
+ */
+struct static_call_site {
+	s32 addr;
+	s32 key;
+};
+
+#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/check.c +127 -1

182 lines not on this page · raw message

tools/objtool/check.h +2 -0
@@ -28,6 +28,7 @@ struct insn_state {
 struct instruction {
 	struct list_head list;
 	struct hlist_node hash;
+	struct list_head static_call_node;
 	struct section *sec;
 	unsigned long offset;
 	unsigned int len;
@@ -51,6 +52,7 @@ struct objtool_file {
 	struct elf *elf;
 	struct list_head insn_list;
 	DECLARE_HASHTABLE(insn_hash, 16);
+	struct list_head static_call_list;
 	bool ignore_unreachables, c_file, hints, rodata;
 };
 
tools/objtool/elf.h +1 -0
@@ -51,6 +51,7 @@ struct symbol {
 	unsigned int len;
 	struct symbol *pfunc, *cfunc, *alias;
 	bool uaccess_safe;
+	bool static_call_tramp;
 };
 
 struct rela {
tools/objtool/sync-check.sh +1 -0
@@ -7,6 +7,7 @@ arch/x86/include/asm/orc_types.h
 arch/x86/include/asm/emulate_prefix.h
 arch/x86/lib/x86-opcode-map.txt
 arch/x86/tools/gen-insn-attr-x86.awk
+include/linux/static_call_types.h
 '
 
 check_2 () {
maintainer
#25 lore raw

Re: [RESEND][PATCH v3 10/17] x86/static_call: Add inline static call implementation for x86-64

Quoted 11 lines · Peter Zijlstra
tools/objtool/check.c +1 -0
@@ -15,6 +15,7 @@
 
 #include <linux/hashtable.h>
 #include <linux/kernel.h>
+#include <linux/static_call_types.h>
 
 #define FAKE_JUMP_OFFSET -1
 

tools/objtool/check.c static int read_retpoline_hints(struct o · line 1352

Quoted 7 lines
tools/objtool/check.c +4 -0
@@ -1345,6 +1346,21 @@ static int read_retpoline_hints(struct o
 	return 0;
 }
 
+static int read_static_call_tramps(struct objtool_file *file)
+{
+	struct section *sec, *sc_sec = find_section_by_name(file->elf, ".static_call.text");
+	struct symbol *func;
	if (!sc_sec)
		return;

no?

I mean, it is enabled by default on X86_64 but not on 32-bit. Or will it be?

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
author maintainer
#26 lore raw

Re: [RESEND][PATCH v3 10/17] x86/static_call: Add inline static call implementation for x86-64

In reply to Peter Zijlstra
Quoted 17 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the inline static call implementation for x86-64. The generated code is identical to the out-of-line case, except we move the trampoline into it's own section.

Then objtool uses the trampoline section to detect all the call sites, which it writes into the .static_call_sites section.

During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.

tools/objtool/check.c +4 -0
+static int read_static_call_tramps(struct objtool_file *file)
+{
+	struct section *sec, *sc_sec = find_section_by_name(file->elf, ".static_call.text");
+	struct symbol *func;

tools/objtool/check.c static int read_retpoline_hints(struct o · line 1362

Quoted 10 lines
tools/objtool/check.c +10 -0
+
+	for_each_sec(file, sec) {
+		list_for_each_entry(func, &sec->symbol_list, list) {
+			if (func->sec == sc_sec)
+				func->static_call_tramp = true;
+		}
+	}
+
+	return 0;
+}

While talking Boris through this stuff, I just realized that if you use a static_call object defined in another translation-unit, IOW. the:

	STATIC_CALL_TRAMP(name)(...);

call is an external call, the above objtool rule will not detect it and it'll stay using the trampoline...

Back to using a naming prefix I suppose.

author maintainer
#27 lore raw

PATCH RESEND v3 11/17 static_call: Simple self-test

In reply to Peter Zijlstra
---
 arch/Kconfig         |    6 ++++++
 kernel/static_call.c |   28 ++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
arch/Kconfig +6 -0
@@ -103,6 +103,12 @@ config STATIC_KEYS_SELFTEST
 	help
 	  Boot time self-test of the branch patching code.
 
+config STATIC_CALL_SELFTEST
+	bool "Static call selftest"
+	depends on HAVE_STATIC_CALL
+	help
+	  Boot time self-test of the call patching code.
+
 config OPTPROBES
 	def_bool y
 	depends on KPROBES && HAVE_OPTPROBES
kernel/static_call.c +28 -0
@@ -364,3 +364,31 @@ static void __init static_call_init(void
 #endif
 }
 early_initcall(static_call_init);
+
+#ifdef CONFIG_STATIC_CALL_SELFTEST
+
+static int func_a(int x)
+{
+	return x+1;
+}
+
+static int func_b(int x)
+{
+	return x+2;
+}
+
+DEFINE_STATIC_CALL(sc_selftest, func_a);
+
+static int __init test_static_call_init(void)
+{
+	WARN_ON(static_call(sc_selftest)(2) != 3);
+	static_call_update(sc_selftest, &func_b);
+	WARN_ON(static_call(sc_selftest)(2) != 4);
+	static_call_update(sc_selftest, &func_a);
+	WARN_ON(static_call(sc_selftest)(2) != 3);
+
+	return 0;
+}
+early_initcall(test_static_call_init);
+
+#endif /* CONFIG_STATIC_CALL_SELFTEST */
maintainer
#28 lore raw

Re: [RESEND][PATCH v3 11/17] static_call: Simple self-test

Quoted 6 lines · Peter Zijlstra
---
 arch/Kconfig         |    6 ++++++
 kernel/static_call.c |   28 ++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

Should we say something?

---

kernel/static_call.c +5 -0
index b7b7fb58afce..2f27cebd5abf 100644
@@ -10,6 +10,9 @@
 #include <linux/processor.h>
 #include <asm/sections.h>
 
+#undef pr_fmt
+#define pr_fmt(fmt) "static_call: " fmt
+
 extern struct static_call_site __start_static_call_sites[],
 			       __stop_static_call_sites[];
 
@@ -381,6 +384,8 @@ DEFINE_STATIC_CALL(sc_selftest, func_a);
 
 static int __init test_static_call_init(void)
 {
+	pr_info("Running static_call selftest... \n");
+
 	WARN_ON(static_call(sc_selftest)(2) != 3);
 	static_call_update(sc_selftest, &func_b);
 	WARN_ON(static_call(sc_selftest)(2) != 4);
Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
author maintainer
#29 lore raw

Re: [RESEND][PATCH v3 11/17] static_call: Simple self-test

Quoted 9 lines · Borislav Petkov
Quoted 6 lines · Peter Zijlstra
---
 arch/Kconfig         |    6 ++++++
 kernel/static_call.c |   28 ++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

Should we say something?

Dunno, that just clutters dmesg, the important bit is it complaining when it goes sideways. But no strong feelings either way.

maintainer
#30 lore raw

Re: [RESEND][PATCH v3 11/17] static_call: Simple self-test

Quoted 2 lines · Peter Zijlstra

Dunno, that just clutters dmesg, the important bit is it complaining when it goes sideways. But no strong feelings either way.

Right, I was playing with it and was wondering whether it ran or not. Whatever you decide - I can always add a printk if I need it to say something.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
author maintainer
#31 lore raw

PATCH RESEND v3 12/17 tracepoint: Optimize using static_call()

In reply to Peter Zijlstra

From: "Steven Rostedt (VMware)" <[email protected]>

Currently the tracepoint site will iterate a vector and issue indirect calls to however many handlers are registered (ie. the vector is long).

Using static_call() it is possible to optimize this for the common case of only having a single handler registered. In this case the static_call() can directly call this handler. Otherwise, if the vector is longer than 1, call a function that iterates the whole vector like the current code.

[peterz: updated to new interface]
---
 include/linux/tracepoint-defs.h |    5 ++
 include/linux/tracepoint.h      |   78 +++++++++++++++++++++++++++-------------
 include/trace/define_trace.h    |   14 +++----
 kernel/tracepoint.c             |   21 ++++++++--
 4 files changed, 82 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +5 -0
@@ -11,6 +11,8 @@
 #include <linux/atomic.h>
 #include <linux/static_key.h>
 
+struct static_call_key;
+
 struct trace_print_flags {
 	unsigned long		mask;
 	const char		*name;
@@ -30,6 +32,9 @@ struct tracepoint_func {
 struct tracepoint {
 	const char *name;		/* Tracepoint name */
 	struct static_key key;
+	struct static_call_key *static_call_key;
+	void *static_call_tramp;
+	void *iterator;
 	int (*regfunc)(void);
 	void (*unregfunc)(void);
 	struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +53 -25

139 lines not on this page · raw message

include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@
 
 #undef TRACE_EVENT
 #define TRACE_EVENT(name, proto, args, tstruct, assign, print)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_CONDITION
 #define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@
 #undef TRACE_EVENT_FN
 #define TRACE_EVENT_FN(name, proto, args, tstruct,		\
 		assign, print, reg, unreg)			\
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_FN_COND
 #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct,		\
 		assign, print, reg, unreg)			\
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_NOP
 #define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@
 
 #undef DEFINE_EVENT
 #define DEFINE_EVENT(template, name, proto, args) \
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_FN
 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_PRINT
 #define DEFINE_EVENT_PRINT(template, name, proto, args, print)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_CONDITION
 #define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
@@ -70,7 +70,7 @@
 
 #undef DECLARE_TRACE
 #define DECLARE_TRACE(name, proto, args)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_INCLUDE
 #undef __TRACE_INCLUDE
kernel/tracepoint.c +17 -4
@@ -221,6 +221,16 @@ static void *func_remove(struct tracepoi
 	return old;
 }
 
+static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
+{
+	void *func = tp->iterator;
+
+	if (!tp_funcs[1].func)
+		func = tp_funcs[0].func;
+
+	__static_call_update(tp->static_call_key, tp->static_call_tramp, func);
+}
+
 /*
  * Add the probe function to a tracepoint.
  */
@@ -251,8 +261,9 @@ static int tracepoint_add_func(struct tr
 	 * include/linux/tracepoint.h using rcu_dereference_sched().
 	 */
 	rcu_assign_pointer(tp->funcs, tp_funcs);
-	if (!static_key_enabled(&tp->key))
-		static_key_slow_inc(&tp->key);
+	tracepoint_update_call(tp, tp_funcs);
+	static_key_enable(&tp->key);
+
 	release_probes(old);
 	return 0;
 }
@@ -281,9 +292,11 @@ static int tracepoint_remove_func(struct
 		if (tp->unregfunc && static_key_enabled(&tp->key))
 			tp->unregfunc();
 
-		if (static_key_enabled(&tp->key))
-			static_key_slow_dec(&tp->key);
+		static_key_disable(&tp->key);
+	} else {
+		tracepoint_update_call(tp, tp_funcs);
 	}
+
 	rcu_assign_pointer(tp->funcs, tp_funcs);
 	release_probes(old);
 	return 0;
author maintainer
#32 lore raw

PATCH RESEND v3 13/17 x86/alternatives: Teach text_poke_bp() to emulate RET

In reply to Peter Zijlstra

Future patches will need to poke a RET instruction, provide the infrastructure required for this.

---
 arch/x86/include/asm/text-patching.h |   17 +++++++++++++++++
 arch/x86/kernel/alternative.c        |    5 +++++
 2 files changed, 22 insertions(+)
arch/x86/include/asm/text-patching.h +17 -0
@@ -53,6 +53,9 @@ extern void text_poke_finish(void);
 #define INT3_INSN_SIZE		1
 #define INT3_INSN_OPCODE	0xCC
 
+#define RET_INSN_SIZE		1
+#define RET_INSN_OPCODE		0xC3
+
 #define CALL_INSN_SIZE		5
 #define CALL_INSN_OPCODE	0xE8
 
@@ -73,6 +76,7 @@ static inline int text_opcode_size(u8 op
 
 	switch(opcode) {
 	__CASE(INT3);
+	__CASE(RET);
 	__CASE(CALL);
 	__CASE(JMP32);
 	__CASE(JMP8);
@@ -138,11 +142,24 @@ static inline void int3_emulate_push(str
 	*(unsigned long *)regs->sp = val;
 }
 
+static inline unsigned long int3_emulate_pop(struct pt_regs *regs)
+{
+	unsigned long val = *(unsigned long *)regs->sp;
+	regs->sp += sizeof(unsigned long);
+	return val;
+}
+
 static inline void int3_emulate_call(struct pt_regs *regs, unsigned long func)
 {
 	int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
 	int3_emulate_jmp(regs, func);
 }
+
+static inline void int3_emulate_ret(struct pt_regs *regs)
+{
+	unsigned long ip = int3_emulate_pop(regs);
+	int3_emulate_jmp(regs, ip);
+}
 #endif /* !CONFIG_UML_X86 */
 
 #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/alternative.c +5 -0
@@ -1045,6 +1045,10 @@ int notrace poke_int3_handler(struct pt_
 		 */
 		goto out_put;
 
+	case RET_INSN_OPCODE:
+		int3_emulate_ret(regs);
+		break;
+
 	case CALL_INSN_OPCODE:
 		int3_emulate_call(regs, (long)ip + tp->rel32);
 		break;
@@ -1187,6 +1191,7 @@ void text_poke_loc_init(struct text_poke
 
 	switch (tp->opcode) {
 	case INT3_INSN_OPCODE:
+	case RET_INSN_OPCODE:
 		break;
 
 	case CALL_INSN_OPCODE:
author maintainer
#33 lore raw

PATCH RESEND v3 14/17 static_call: Add static_cond_call()

In reply to Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

	if (func_ptr)
		func_ptr(args...)

For the trampoline (which is in effect a tail-call), we patch the JMP.d32 into a RET, which then directly consumes the trampoline call.

For the in-line sites we replace the CALL with a NOP5.

---
 arch/x86/include/asm/static_call.h |   10 ++++++++
 arch/x86/kernel/static_call.c      |   42 ++++++++++++++++++++++++++++---------
 include/linux/static_call.h        |   29 +++++++++++++++++++++++++
 3 files changed, 71 insertions(+), 10 deletions(-)
arch/x86/include/asm/static_call.h +10 -0
@@ -32,4 +32,14 @@
 	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
 	    ".popsection					\n")
 
+#define ARCH_DEFINE_STATIC_CALL_RETTRAMP(name)				\
+	asm(".pushsection .static_call.text, \"ax\"		\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(name) "		\n"	\
+	    STATIC_CALL_TRAMP_STR(name) ":			\n"	\
+	    "	ret; nop; nop; nop; nop;			\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(name) ", @function	\n"	\
+	    ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+	    ".popsection					\n")
+
 #endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/static_call.c +32 -10

61 lines not on this page · raw message

include/linux/static_call.h +29 -0
@@ -17,6 +17,7 @@
  *   DECLARE_STATIC_CALL(name, func);
  *   DEFINE_STATIC_CALL(name, func);
  *   static_call(name)(args...);
+ *   static_cond_call(name)(args...)
  *   static_call_update(name, func);
  *
  * Usage example:
@@ -107,7 +108,17 @@ extern int static_call_text_reserved(voi
 	__ADDRESSABLE(STATIC_CALL_NAME(name));				\
 	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = NULL,						\
+		.type = 1,						\
+	};								\
+	__ADDRESSABLE(STATIC_CALL_NAME(name));				\
+	ARCH_DEFINE_STATIC_CALL_RETTRAMP(name)
+
 #define static_call(name)	STATIC_CALL_TRAMP(name)
+#define static_cond_call(name)	STATIC_CALL_TRAMP(name)
 
 #define EXPORT_STATIC_CALL(name)					\
 	EXPORT_SYMBOL(STATIC_CALL_NAME(name));				\
@@ -130,7 +141,15 @@ struct static_call_key {
 	};								\
 	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = NULL,						\
+	};								\
+	ARCH_DEFINE_STATIC_CALL_RETTRAMP(name)
+
 #define static_call(name)	STATIC_CALL_TRAMP(name)
+#define static_cond_call(name)	STATIC_CALL_TRAMP(name)
 
 static inline
 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
@@ -161,9 +180,19 @@ struct static_call_key {
 		.func = _func,						\
 	}
 
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = NULL,						\
+	}
+
 #define static_call(name)						\
 	((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
 
+#define static_cond_call(name)						\
+	if (STATIC_CALL_NAME(name).func)				\
+		((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
+
 static inline
 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
 {
#34 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 6 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

        if (func_ptr)
                func_ptr(args...)

Is there any reason why this shouldn't be the default static call pattern?

IOW, do we need the special "cond" versions at all? Couldn't we just say that this is how static calls fundamentally work - if the function is NULL, they are nops?

Linus

#35 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 8 lines · Linus Torvalds
Quoted 4 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

Quoted 10 lines
Quoted 2 lines
       if (func_ptr)
               func_ptr(args...)

Is there any reason why this shouldn't be the default static call pattern?

IOW, do we need the special "cond" versions at all? Couldn't we just say that this is how static calls fundamentally work - if the function is NULL, they are nops?

I haven’t checked if static calls currently support return values, but the conditional case only makes sense for functions that return void.

Aside from that, it might be nice for passing NULL in to warn or bug when the NULL pointer is stored instead of silently NOPping out the call in cases where having a real implementation isn’t optional.

#36 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 7 lines · Andy Lutomirski

I haven’t checked if static calls currently support return values, but the conditional case only makes sense for functions that return void.

Aside from that, it might be nice for passing NULL in to warn or bug when the NULL pointer is stored instead of silently NOPping out the call in cases where having a real implementation isn’t optional.

Both good points. I take back my question.

And it aside from warning about passing in NULL then it doesn't work, I wonder if we could warn - at build time - when then using the COND version with a function that doesn't return void?

Of course, one alternative is to just say "instead of using NOP, use 'xorl %eax,%eax'", and then we'd have the rule that a NULL conditional function returns zero (or NULL).

I _think_ a "xorl %eax,%eax ; retq" is just three bytes and would fit in the tailcall slot too.

Linus

author maintainer
#37 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 9 lines · Linus Torvalds
Quoted 7 lines · Andy Lutomirski

I haven’t checked if static calls currently support return values, but the conditional case only makes sense for functions that return void.

Aside from that, it might be nice for passing NULL in to warn or bug when the NULL pointer is stored instead of silently NOPping out the call in cases where having a real implementation isn’t optional.

Quoted 5 lines

Both good points. I take back my question.

And it aside from warning about passing in NULL then it doesn't work, I wonder if we could warn - at build time - when then using the COND version with a function that doesn't return void?

I actually (abuse) do that in the last patch... the reason being that DEFINE_STATIC_COND_CALL() ends up only needing a type expression for the second argument, while DEFINE_STATIC_CALL() needs an actual function.

Quoted 6 lines

Of course, one alternative is to just say "instead of using NOP, use 'xorl %eax,%eax'", and then we'd have the rule that a NULL conditional function returns zero (or NULL).

I _think_ a "xorl %eax,%eax ; retq" is just three bytes and would fit in the tailcall slot too.

Correct. The only problem is that our text patching machinery can't replace multiple instructions :/

author maintainer
#38 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Linus Torvalds

Of course, one alternative is to just say "instead of using NOP, use 'xorl %eax,%eax'", and then we'd have the rule that a NULL conditional function returns zero (or NULL).

I _think_ a "xorl %eax,%eax ; retq" is just three bytes and would fit in the tailcall slot too.

Correct. The only problem is that our text patching machinery can't replace multiple instructions :/

To clarify; the problem is a task getting preempted with its RIP at the RET. Then when we rewrite the text to be a CALL/JMP.d32 it will read garbage (1 byte into the displacement of the instruction) instead of a RET when it resumes.

Now, there are ways to fix this, the easiest being calling synchronize_rcu_tasks() just like optprobes does (see also commit 5c02ece81848 ("x86/kprobes: Fix ordering while text-patching")).

It would mean patching a call away from NULL will be 'expensive' but it ought to work.

I'll try and do the patch, see what it looks like.

#39 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 5 lines · Peter Zijlstra

To clarify; the problem is a task getting preempted with its RIP at the RET. Then when we rewrite the text to be a CALL/JMP.d32 it will read garbage (1 byte into the displacement of the instruction) instead of a RET when it resumes.

Yeah, I realized it when you mentioned it. I was trying to come up with some clever way to avoid it, but can't see anything.

I can come up with insane models - you could replace the xor;ret sequence with a jump to a trampoline where you've aligned the target trampoline so that the third byte in the "jmp xxx" remains a 'ret' instruction. Then replace _that_ one with a "call_rcu()" callback. Wild handwaving of "I'm sure this could be made to work".

But nothing remotely sane comes to mind.

Linus

#40 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

In reply to Linus Torvalds
Quoted 17 lines · Linus Torvalds
Quoted 2 lines · Andy Lutomirski

I haven’t checked if static calls currently support return values,

but

Quoted 5 lines

the conditional case only makes sense for functions that return void.

Aside from that, it might be nice for passing NULL in to warn or bug when the NULL pointer is stored instead of silently NOPping out the call in cases where having a real implementation isn’t optional.

Both good points. I take back my question.

And it aside from warning about passing in NULL then it doesn't work, I wonder if we could warn - at build time - when then using the COND version with a function that doesn't return void?

Quoted 8 lines

Of course, one alternative is to just say "instead of using NOP, use 'xorl %eax,%eax'", and then we'd have the rule that a NULL conditional function returns zero (or NULL).

I _think_ a "xorl %eax,%eax ; retq" is just three bytes and would fit in the tailcall slot too.

Linus

"movl $0,%eax" is five bytes, the same length as a call. Doesn't work for a tailcall, still, although if the sequence:

jmp tailcall
retq

... can be generated at the tailcall site then the jmp can get patched out.

This would be equivalent to disabling tailcalls except that the stack frame is normally not unwound until between the call and the ret, so just disabling tailcalls from the compiler pov doesn't work.

Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.
#41 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 7 lines · On Wed, Mar 25, 2020 at 12:35 PM <[email protected]> wrote:

"movl $0,%eax" is five bytes, the same length as a call. Doesn't work for a tailcall, still, although if the sequence:

jmp tailcall
retq

... can be generated at the tailcall site then the jmp can get patched out.

No, the problem is literally that the whole approach depends on the compiler just generating normal code for the static calls.

And the tailcall is the only interesting case. The normal call thing can be trivially just a single instruction (a mov like you say, but also easily just a xor padded with prefixes).

Linus

author maintainer
#42 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 7 lines · Linus Torvalds
Quoted 6 lines · On Wed, Mar 25, 2020 at 12:35 PM <[email protected]> wrote:

"movl $0,%eax" is five bytes, the same length as a call. Doesn't work for a tailcall, still, although if the sequence:

jmp tailcall
retq

Quoted 8 lines
Quoted 1 line

... can be generated at the tailcall site then the jmp can get patched out.

No, the problem is literally that the whole approach depends on the compiler just generating normal code for the static calls.

And the tailcall is the only interesting case. The normal call thing can be trivially just a single instruction (a mov like you say, but also easily just a xor padded with prefixes).

So I got the text poking bit written, and that turned out to be the simple part :/ Find below.

Then we can do:

#define static_void_call(name)
	if (STATIC_CALL_NAME(name).func) \
		((typeof(STATIC_CALL_TRAMP(name))*)STATIC_CALL_NAME(name).func)

Which works, as evidenced by if being the current static_cond_call(), but it is non-optimal code-gen for the case where func will never be NULL, and also there is no way to write a !void version of the same.

The best I can come up with is something like:

#define static_call(name, args...) ({ \
	typeof(STATIC_CALL_TRAMP(name)(args)) __ret = (typeof(STATIC_CALL_TRAMP(name)(args)))0; \
	if (STATIC_CALL_NAME(name).func) \
		__ret = ((typeof(STATIC_CALL_TRAMP(name))*)STATIC_CALL_NAME(name).func)(args); \
	__ret; })

Which has a different (and IMO less natural) syntax.

That then brings us to the HAVE_STATIC_CALL variant; there we need to somehow make the void vs !void thing persistent, and there I ran out of ideas.

Initially I figured we could do something like:

#define annotate_void_call() ({ \
	asm volatile("%c0:\n\t" \
		     ".pushsection .discard.void_call\n\t" \
		     ".long %c0b - .\n\t" \
		     ".popsection\n\t" : : "i" (__COUNTER__)); \
})
#define static_void_call(name) \
	annotate_void_call(); \
	STATIC_CALL_TRAMP(name)

But that doesn't actually work for something like:

	static_void_call(foo)(static_call(bar)());

Where the argument setup of the call, include another static call. Arguably this is quite insane, and we could just say:
"don't-do-that-then", but it does show how fragile this is.

Anyway, let me ponder this a little more... brain is starting to give out anyway. More tomorrow.

---

arch/x86/kernel/alternative.c +34 -2
index 346c98d5261e..240996338f66 100644
@@ -947,6 +947,7 @@ struct text_poke_loc {
 	s32 rel32;
 	u8 opcode;
 	const u8 text[POKE_MAX_OPCODE_SIZE];
+	u8 multi;
 };
 
 struct bp_patching_desc {
@@ -1103,8 +1104,8 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 		.refs = ATOMIC_INIT(1),
 	};
 	unsigned char int3 = INT3_INSN_OPCODE;
+	int do_sync, do_multi = 0;
 	unsigned int i;
-	int do_sync;
 
 	lockdep_assert_held(&text_mutex);
 
@@ -1119,11 +1120,24 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 	/*
 	 * First step: add a int3 trap to the address that will be patched.
 	 */
-	for (i = 0; i < nr_entries; i++)
+	for (i = 0; i < nr_entries; i++) {
 		text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE);
+		do_multi |= tp[i].multi;
+	}
 
 	text_poke_sync();
 
+	if (do_multi) {
+		/*
+		 * In case the 'old' text consisted of multiple instructions
+		 * we need to wait for an rcu_tasks quiescence period to ensure
+		 * all potentially preempted tasks have normally scheduled.
+		 * This ensures no tasks still have their instruction pointer
+		 * pointed at what will become the middle of an instruction.
+		 */
+		synchronize_rcu_tasks();
+	}
+
 	/*
 	 * Second step: update all but the first byte of the patched range.
 	 */
@@ -1176,10 +1190,28 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
 {
 	struct insn insn;
 
+	/*
+	 * Determine if the 'old' text at @addr consists of multiple
+	 * instructions. Make an exception for INT3 and RET, since
+	 * they don't (necessarily) continue to execute the following
+	 * instructions.
+	 */
+	kernel_insn_init(&insn, addr, MAX_INSN_SIZE);
+	insn_get_length(&insn);
+	tp.multi = (insn.legnth < len) &&
+		   (insn.opcode.bytes[0] != RET_INSN_OPCODE ||
+		    insn.opcode.bytes[0] != INT3_INSN_OPCODE);
+
+	/*
+	 * Copy the 'new' text into the text_poke vector.
+	 */
 	memcpy((void *)tp->text, opcode, len);
 	if (!emulate)
 		emulate = opcode;
 
+	/*
+	 * Decode the instruction poke_int3_handler() needs to emulate.
+	 */
 	kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
 	insn_get_length(&insn);
 
arch/x86/kernel/static_call.c +13 -0
index 1e82e2486e76..2055e2d3674d 100644
@@ -9,8 +9,13 @@ enum insn_type {
 	nop = 1,  /* site cond-call */
 	jmp = 2,  /* tramp / site tail-call */
 	ret = 3,  /* tramp / site cond-tail-call */
+	null = 4,
+	null_ret = 5,
 };
 
+static const u8 null_insn[5] =     { 0xb8, 0x00, 0x00, 0x00, 0x00 }; /* movl $0, %eax */
+static const u8 null_ret_insn[5] = { 0x31, 0xc0, 0xc3, 0x90, 0x90 }; /* xorl %eax, %eax; ret; nop; nop; */
+
 static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
 {
 	int size = CALL_INSN_SIZE;
@@ -34,6 +39,14 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void
 		size = RET_INSN_SIZE;
 		break;
 
+	case null:
+		code = null_insi;
+		break;
+
+	case null_ret:
+		code = null_ret_insn;
+		break;
+
 	default: /* GCC is a moron -- it figures @code can be uninitialized below */
 		BUG();
 	}
author maintainer
#43 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

In reply to Linus Torvalds
Quoted 5 lines · Linus Torvalds
Quoted 4 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

Quoted 8 lines
Quoted 2 lines
        if (func_ptr)
                func_ptr(args...)

Is there any reason why this shouldn't be the default static call pattern?

IOW, do we need the special "cond" versions at all? Couldn't we just say that this is how static calls fundamentally work - if the function is NULL, they are nops?

That doesn't work for functions that have a return value ...

#44 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

	if (func_ptr)
		func_ptr(args...)
include/linux/static_call.h +3 -0
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\

include/linux/static_call.h struct static_call_key { · line 195

Quoted 10 lines
include/linux/static_call.h +7 -0
+		.func = NULL,						\
+	}
+
 #define static_call(name)						\
 	((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
 
+#define static_cond_call(name)						\
+	if (STATIC_CALL_NAME(name).func)				\
+		((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
+

What, apart from fear of being ridiculed by kernel folks, prevents the compiler from reloading STATIC_CALL_NAME(name).func ? IOW, doesn't this want a READ_ONCE somewhere?

And please remind me, what is the consensus for sizeof(long) loads: does static_call() need load-tearing protection or not?

Rasmus

author maintainer
#45 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 17 lines · Rasmus Villemoes
Quoted 16 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

	if (func_ptr)
		func_ptr(args...)
include/linux/static_call.h +6 -0
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = NULL,						\
+	}
+
 #define static_call(name)						\
 	((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
Quoted 8 lines

include/linux/static_call.h struct static_call_key { · line 195

Quoted 4 lines
include/linux/static_call.h +4 -0
+#define static_cond_call(name)						\
+	if (STATIC_CALL_NAME(name).func)				\
+		((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
+

What, apart from fear of being ridiculed by kernel folks, prevents the compiler from reloading STATIC_CALL_NAME(name).func ? IOW, doesn't this want a READ_ONCE somewhere?

Hurmph.. I suspect you're quite right, but at the same time I can't seem to write a macro that does that :/ Let me try harder.

Quoted 2 lines

And please remind me, what is the consensus for sizeof(long) loads: does static_call() need load-tearing protection or not?

We all like to believe compilers are broken when they tear naturally aligned words, but we're also not quite comfortable trusting that.

#46 lore raw

Re: [RESEND][PATCH v3 14/17] static_call: Add static_cond_call()

Quoted 27 lines · Peter Zijlstra
Quoted 17 lines · Rasmus Villemoes
Quoted 16 lines · Peter Zijlstra

Extend the static_call infrastructure to optimize the following common pattern:

	if (func_ptr)
		func_ptr(args...)
include/linux/static_call.h +6 -0
+#define DEFINE_STATIC_COND_CALL(name, _func)				\
+	DECLARE_STATIC_CALL(name, _func);				\
+	struct static_call_key STATIC_CALL_NAME(name) = {		\
+		.func = NULL,						\
+	}
+
 #define static_call(name)						\
 	((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
Quoted 8 lines

include/linux/static_call.h struct static_call_key { · line 195

Quoted 4 lines
include/linux/static_call.h +4 -0
+#define static_cond_call(name)						\
+	if (STATIC_CALL_NAME(name).func)				\
+		((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func))
+

What, apart from fear of being ridiculed by kernel folks, prevents the compiler from reloading STATIC_CALL_NAME(name).func ? IOW, doesn't this want a READ_ONCE somewhere?

Quoted 2 lines

Hurmph.. I suspect you're quite right, but at the same time I can't seem to write a macro that does that :/ Let me try harder.

Hm, yeah, essentially one wants some macro magic that turns

foo(a)(b, c, d)

into

bar(a, b, c, d)

and then bar() can do the right thing.

One option is to give up on the nice syntax and just make it

static_cond_call(func, ...)

But, here's another few things that makes me wonder if the cond_call variant is worth it, at least in its current form: In the case where !ARCH_HAVE_STATIC_CALL, so static_cond_call(foo)(a, b, c) is just syntax sugar for

if (foo)
foo(a, b, c)

gcc can choose to wait with computing the argument expressions a, b, c until after the test - they may be somewhat expensive, but at the very least there's some register shuffling to do to prepare for the call, and probably also some restoring afterwards. In the ARCH_HAVE_STATIC_CALL case, whether inline or not, it becomes an unconditional call from gcc's perspective, so all the arguments must be computed and stuffed in the right registers. That price may be higher than the load+test. Not to mention the fact that side-effects in the arguments happen unconditionally for ARCH_HAVE_STATIC_CALL but only if func is non-null for !ARCH_HAVE_STATIC_CALL.

Perhaps associating a static_key with each STATIC_COND_CALL could solve these. But that of course makes the update procedure somewhat more complicated.

Rasmus

author maintainer
#47 lore raw

PATCH RESEND v3 15/17 static_call: Handle tail-calls

In reply to Peter Zijlstra

GCC can turn our static_call(name)(args...) into a tail call, in which case we get a JMP.d32 into the trampoline (which then does a further tail-call).

Teach objtool to recognise and mark these in .static_call_sites and adjust the code patching to deal with this.

---
 arch/x86/kernel/static_call.c           |    4 ++--
 include/linux/static_call.h             |    4 ++--
 include/linux/static_call_types.h       |    7 +++++++
 kernel/static_call.c                    |   21 +++++++++++++--------
 tools/include/linux/static_call_types.h |    7 +++++++
 tools/objtool/check.c                   |   18 +++++++++++++-----
 6 files changed, 44 insertions(+), 17 deletions(-)
arch/x86/kernel/static_call.c +2 -2
@@ -41,7 +41,7 @@ static void __static_call_transform(void
 	text_poke_bp(insn, code, size, NULL);
 }
 
-void arch_static_call_transform(void *site, void *tramp, void *func)
+void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
 {
 	mutex_lock(&text_mutex);
 
@@ -49,7 +49,7 @@ void arch_static_call_transform(void *si
 		__static_call_transform(tramp, jmp + !func, func);
 
 	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site)
-		__static_call_transform(site, !func, func);
+		__static_call_transform(site, 2*tail + !func, func);
 
 	mutex_unlock(&text_mutex);
 }
include/linux/static_call.h +2 -2
@@ -64,7 +64,7 @@
 /*
  * Either @site or @tramp can be NULL.
  */
-extern void arch_static_call_transform(void *site, void *tramp, void *func);
+extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
 #else
 #define STATIC_CALL_TRAMP_ADDR(name) NULL
@@ -140,7 +140,7 @@ void __static_call_update(struct static_
 {
 	cpus_read_lock();
 	WRITE_ONCE(key->func, func);
-	arch_static_call_transform(NULL, tramp, func);
+	arch_static_call_transform(NULL, tramp, func, false);
 	cpus_read_unlock();
 }
 
include/linux/static_call_types.h +7 -0
@@ -14,6 +14,13 @@
 #define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
 
 /*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL	/* tail call */
+#define STATIC_CALL_SITE_INIT 2UL	/* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
  * The static call site table needs to be created by external tooling (objtool
  * or a compiler plugin).
  */
kernel/static_call.c +13 -8
@@ -15,8 +15,6 @@ extern struct static_call_site __start_s
 
 static bool static_call_initialized;
 
-#define STATIC_CALL_INIT 1UL
-
 /* mutex to protect key modules/sites */
 static DEFINE_MUTEX(static_call_mutex);
 
@@ -39,18 +37,23 @@ static inline void *static_call_addr(str
 static inline struct static_call_key *static_call_key(const struct static_call_site *site)
 {
 	return (struct static_call_key *)
-		(((long)site->key + (long)&site->key) & ~STATIC_CALL_INIT);
+		(((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);
 }
 
 /* These assume the key is word-aligned. */
 static inline bool static_call_is_init(struct static_call_site *site)
 {
-	return ((long)site->key + (long)&site->key) & STATIC_CALL_INIT;
+	return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;
+}
+
+static inline bool static_call_is_tail(struct static_call_site *site)
+{
+	return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;
 }
 
 static inline void static_call_set_init(struct static_call_site *site)
 {
-	site->key = ((long)static_call_key(site) | STATIC_CALL_INIT) -
+	site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -
 		    (long)&site->key;
 }
 
@@ -104,7 +107,7 @@ void __static_call_update(struct static_
 
 	key->func = func;
 
-	arch_static_call_transform(NULL, tramp, func);
+	arch_static_call_transform(NULL, tramp, func, false);
 
 	/*
 	 * If uninitialized, we'll not update the callsites, but they still
@@ -153,7 +156,8 @@ void __static_call_update(struct static_
 				continue;
 			}
 
-			arch_static_call_transform(site_addr, NULL, func);
+			arch_static_call_transform(site_addr, NULL, func,
+				static_call_is_tail(site));
 		}
 	}
 
@@ -197,7 +201,8 @@ static int __static_call_init(struct mod
 			key->next = site_mod;
 		}
 
-		arch_static_call_transform(site_addr, NULL, key->func);
+		arch_static_call_transform(site_addr, NULL, key->func,
+				static_call_is_tail(site));
 	}
 
 	return 0;
tools/include/linux/static_call_types.h +7 -0
@@ -14,6 +14,13 @@
 #define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
 
 /*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL	/* tail call */
+#define STATIC_CALL_SITE_INIT 2UL	/* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
  * The static call site table needs to be created by external tooling (objtool
  * or a compiler plugin).
  */
tools/objtool/check.c +13 -5
@@ -585,6 +585,10 @@ static int add_jump_destinations(struct
 		} else {
 			/* external sibling call */
 			insn->call_dest = rela->sym;
+			if (insn->call_dest->static_call_tramp) {
+				list_add_tail(&insn->static_call_node,
+					      &file->static_call_list);
+			}
 			continue;
 		}
 
@@ -636,6 +640,10 @@ static int add_jump_destinations(struct
 
 				/* internal sibling call */
 				insn->call_dest = insn->jump_dest->func;
+				if (insn->call_dest->static_call_tramp) {
+					list_add_tail(&insn->static_call_node,
+						      &file->static_call_list);
+				}
 			}
 		}
 	}
@@ -1348,6 +1356,10 @@ static int decode_sections(struct objtoo
 	if (ret)
 		return ret;
 
+	ret = read_static_call_tramps(file);
+	if (ret)
+		return ret;
+
 	ret = add_jump_destinations(file);
 	if (ret)
 		return ret;
@@ -1372,10 +1384,6 @@ static int decode_sections(struct objtoo
 	if (ret)
 		return ret;
 
-	ret = read_static_call_tramps(file);
-	if (ret)
-		return ret;
-
 	return 0;
 }
 
@@ -2505,7 +2513,7 @@ static int create_static_call_sections(s
 		}
 		memset(rela, 0, sizeof(*rela));
 		rela->sym = key_sym;
-		rela->addend = 0;
+		rela->addend = is_sibling_call(insn) ? STATIC_CALL_SITE_TAIL : 0;
 		rela->type = R_X86_64_PC32;
 		rela->offset = idx * sizeof(struct static_call_site) + 4;
 		list_add_tail(&rela->list, &rela_sec->rela_list);
author maintainer
#48 lore raw

PATCH RESEND v3 16/17 static_call: Allow early init

In reply to Peter Zijlstra

In order to use static_call() to wire up x86_pmu, we need to initialize earlier; copy some of the tricks from jump_label to enable this.

Primarily we overload key->next to store a sites pointer when there are no modules, this avoids having to use kmalloc() to initialize the sites and allows us to run much earlier.

(arguably, this is much much earlier than needed for perf, but it might allow other uses.)

---
 arch/x86/kernel/setup.c       |    2 +
 arch/x86/kernel/static_call.c |    8 +++++-
 include/linux/static_call.h   |   15 ++++++++++--
 kernel/static_call.c          |   52 +++++++++++++++++++++++++++++++++++++++---
 4 files changed, 71 insertions(+), 6 deletions(-)
arch/x86/kernel/setup.c +2 -0
@@ -18,6 +18,7 @@
 #include <linux/sfi.h>
 #include <linux/tboot.h>
 #include <linux/usb/xhci-dbgp.h>
+#include <linux/static_call.h>
 
 #include <uapi/linux/mount.h>
 
@@ -841,6 +842,7 @@ void __init setup_arch(char **cmdline_p)
 	early_cpu_init();
 	arch_init_ideal_nops();
 	jump_label_init();
+	static_call_init();
 	early_ioremap_init();
 
 	setup_olpc_ofw_pgd();
arch/x86/kernel/static_call.c +7 -1
@@ -11,7 +11,7 @@ enum insn_type {
 	ret = 3,  /* tramp / site cond-tail-call */
 };
 
-static void __static_call_transform(void *insn, enum insn_type type, void *func)
+static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
 {
 	int size = CALL_INSN_SIZE;
 	const void *code;
@@ -33,11 +33,17 @@ static void __static_call_transform(void
 		code = text_gen_insn(RET_INSN_OPCODE, insn, func);
 		size = RET_INSN_SIZE;
 		break;
+
+	default: /* GCC is a moron -- it figures @code can be uninitialized below */
+		BUG();
 	}
 
 	if (memcmp(insn, code, size) == 0)
 		return;
 
+	if (unlikely(system_state == SYSTEM_BOOTING))
+		return text_poke_early(insn, code, size);
+
 	text_poke_bp(insn, code, size, NULL);
 }
 
include/linux/static_call.h +13 -2
@@ -84,6 +84,8 @@ extern void arch_static_call_transform(v
 
 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
 
+extern void __init static_call_init(void);
+
 struct static_call_mod {
 	struct static_call_mod *next;
 	struct module *mod; /* for vmlinux, mod == NULL */
@@ -92,7 +94,12 @@ struct static_call_mod {
 
 struct static_call_key {
 	void *func;
-	struct static_call_mod *next;
+	union {
+		/* bit0 => 0 - next, 1 - sites */
+		unsigned long type;
+		struct static_call_mod *next;
+		struct static_call_site *sites;
+	};
 };
 
 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
@@ -103,7 +110,7 @@ extern int static_call_text_reserved(voi
 	DECLARE_STATIC_CALL(name, _func);				\
 	struct static_call_key STATIC_CALL_NAME(name) = {		\
 		.func = _func,						\
-		.next = NULL,						\
+		.type = 1,						\
 	};								\
 	__ADDRESSABLE(STATIC_CALL_NAME(name));				\
 	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
@@ -121,6 +128,8 @@ extern int static_call_text_reserved(voi
 
 #elif defined(CONFIG_HAVE_STATIC_CALL)
 
+static inline void static_call_init(void) { }
+
 struct static_call_key {
 	void *func;
 };
@@ -154,6 +163,8 @@ static inline int static_call_text_reser
 
 #else /* Generic implementation */
 
+static inline void static_call_init(void) { }
+
 struct static_call_key {
 	void *func;
 };
kernel/static_call.c +49 -3

89 lines not on this page · raw message

author maintainer
#49 lore raw

PATCH RESEND v3 17/17 x86/perf, static_call: Optimize x86_pmu methods

In reply to Peter Zijlstra

Replace many of the indirect calls with static_call().

XXX run performance numbers

---
 arch/x86/events/core.c |  130 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 90 insertions(+), 40 deletions(-)
arch/x86/events/core.c +90 -40
@@ -28,6 +28,7 @@
 #include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/nospec.h>
+#include <linux/static_call.h>
 
 #include <asm/apic.h>
 #include <asm/stacktrace.h>
@@ -52,6 +53,30 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpu
 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
 
+DEFINE_STATIC_COND_CALL(x86_pmu_handle_irq,  *x86_pmu.handle_irq);
+DEFINE_STATIC_COND_CALL(x86_pmu_disable_all, *x86_pmu.disable_all);
+DEFINE_STATIC_COND_CALL(x86_pmu_enable_all,  *x86_pmu.enable_all);
+DEFINE_STATIC_COND_CALL(x86_pmu_enable,	     *x86_pmu.enable);
+DEFINE_STATIC_COND_CALL(x86_pmu_disable,     *x86_pmu.disable);
+
+DEFINE_STATIC_COND_CALL(x86_pmu_add,  *x86_pmu.add);
+DEFINE_STATIC_COND_CALL(x86_pmu_del,  *x86_pmu.del);
+DEFINE_STATIC_COND_CALL(x86_pmu_read, *x86_pmu.read);
+
+DEFINE_STATIC_COND_CALL(x86_pmu_schedule_events,       *x86_pmu.schedule_events);
+DEFINE_STATIC_COND_CALL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints);
+DEFINE_STATIC_COND_CALL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints);
+
+DEFINE_STATIC_COND_CALL(x86_pmu_start_scheduling,  *x86_pmu.start_scheduling);
+DEFINE_STATIC_COND_CALL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
+DEFINE_STATIC_COND_CALL(x86_pmu_stop_scheduling,   *x86_pmu.stop_scheduling);
+
+DEFINE_STATIC_COND_CALL(x86_pmu_sched_task,    *x86_pmu.sched_task);
+DEFINE_STATIC_COND_CALL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);
+
+DEFINE_STATIC_COND_CALL(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
+DEFINE_STATIC_COND_CALL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
+
 u64 __read_mostly hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
@@ -660,7 +685,7 @@ static void x86_pmu_disable(struct pmu *
 	cpuc->enabled = 0;
 	barrier();
 
-	x86_pmu.disable_all();
+	static_call(x86_pmu_disable_all)();
 }
 
 void x86_pmu_enable_all(int added)
@@ -907,8 +932,7 @@ int x86_schedule_events(struct cpu_hw_ev
 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
 		n0 -= cpuc->n_txn;
 
-	if (x86_pmu.start_scheduling)
-		x86_pmu.start_scheduling(cpuc);
+	static_cond_call(x86_pmu_start_scheduling)(cpuc);
 
 	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 		c = cpuc->event_constraint[i];
@@ -925,7 +949,7 @@ int x86_schedule_events(struct cpu_hw_ev
 		 * change due to external factors (sibling state, allow_tfa).
 		 */
 		if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
-			c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
+			c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]);
 			cpuc->event_constraint[i] = c;
 		}
 
@@ -1008,8 +1032,7 @@ int x86_schedule_events(struct cpu_hw_ev
 	if (!unsched && assign) {
 		for (i = 0; i < n; i++) {
 			e = cpuc->event_list[i];
-			if (x86_pmu.commit_scheduling)
-				x86_pmu.commit_scheduling(cpuc, i, assign[i]);
+			static_cond_call(x86_pmu_commit_scheduling)(cpuc, i, assign[i]);
 		}
 	} else {
 		for (i = n0; i < n; i++) {
@@ -1018,15 +1041,13 @@ int x86_schedule_events(struct cpu_hw_ev
 			/*
 			 * release events that failed scheduling
 			 */
-			if (x86_pmu.put_event_constraints)
-				x86_pmu.put_event_constraints(cpuc, e);
+			static_cond_call(x86_pmu_put_event_constraints)(cpuc, e);
 
 			cpuc->event_constraint[i] = NULL;
 		}
 	}
 
-	if (x86_pmu.stop_scheduling)
-		x86_pmu.stop_scheduling(cpuc);
+	static_cond_call(x86_pmu_stop_scheduling)(cpuc);
 
 	return unsched ? -EINVAL : 0;
 }
@@ -1217,7 +1238,7 @@ static void x86_pmu_enable(struct pmu *p
 	cpuc->enabled = 1;
 	barrier();
 
-	x86_pmu.enable_all(added);
+	static_call(x86_pmu_enable_all)(added);
 }
 
 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -1338,7 +1359,7 @@ static int x86_pmu_add(struct perf_event
 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
 		goto done_collect;
 
-	ret = x86_pmu.schedule_events(cpuc, n, assign);
+	ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
 	if (ret)
 		goto out;
 	/*
@@ -1356,13 +1377,11 @@ static int x86_pmu_add(struct perf_event
 	cpuc->n_added += n - n0;
 	cpuc->n_txn += n - n0;
 
-	if (x86_pmu.add) {
-		/*
-		 * This is before x86_pmu_enable() will call x86_pmu_start(),
-		 * so we enable LBRs before an event needs them etc..
-		 */
-		x86_pmu.add(event);
-	}
+	/*
+	 * This is before x86_pmu_enable() will call x86_pmu_start(),
+	 * so we enable LBRs before an event needs them etc..
+	 */
+	static_cond_call(x86_pmu_add)(event);
 
 	ret = 0;
 out:
@@ -1390,7 +1409,7 @@ static void x86_pmu_start(struct perf_ev
 	cpuc->events[idx] = event;
 	__set_bit(idx, cpuc->active_mask);
 	__set_bit(idx, cpuc->running);
-	x86_pmu.enable(event);
+	static_call(x86_pmu_enable)(event);
 	perf_event_update_userpage(event);
 }
 
@@ -1460,7 +1479,7 @@ void x86_pmu_stop(struct perf_event *eve
 	struct hw_perf_event *hwc = &event->hw;
 
 	if (test_bit(hwc->idx, cpuc->active_mask)) {
-		x86_pmu.disable(event);
+		static_call(x86_pmu_disable)(event);
 		__clear_bit(hwc->idx, cpuc->active_mask);
 		cpuc->events[hwc->idx] = NULL;
 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
@@ -1510,8 +1529,7 @@ static void x86_pmu_del(struct perf_even
 	if (i >= cpuc->n_events - cpuc->n_added)
 		--cpuc->n_added;
 
-	if (x86_pmu.put_event_constraints)
-		x86_pmu.put_event_constraints(cpuc, event);
+	static_cond_call(x86_pmu_put_event_constraints)(cpuc, event);
 
 	/* Delete the array entry. */
 	while (++i < cpuc->n_events) {
@@ -1524,13 +1542,12 @@ static void x86_pmu_del(struct perf_even
 	perf_event_update_userpage(event);
 
 do_del:
-	if (x86_pmu.del) {
-		/*
-		 * This is after x86_pmu_stop(); so we disable LBRs after any
-		 * event can need them etc..
-		 */
-		x86_pmu.del(event);
-	}
+
+	/*
+	 * This is after x86_pmu_stop(); so we disable LBRs after any
+	 * event can need them etc..
+	 */
+	static_cond_call(x86_pmu_del)(event);
 }
 
 int x86_pmu_handle_irq(struct pt_regs *regs)
@@ -1608,7 +1625,7 @@ perf_event_nmi_handler(unsigned int cmd,
 		return NMI_DONE;
 
 	start_clock = sched_clock();
-	ret = x86_pmu.handle_irq(regs);
+	ret = static_call(x86_pmu_handle_irq)(regs);
 	finish_clock = sched_clock();
 
 	perf_sample_event_took(finish_clock - start_clock);
@@ -1821,6 +1838,38 @@ ssize_t x86_event_sysfs_show(char *page,
 static struct attribute_group x86_pmu_attr_group;
 static struct attribute_group x86_pmu_caps_group;
 
+static void x86_pmu_static_call_update(void)
+{
+	static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq);
+	static_call_update(x86_pmu_disable_all, x86_pmu.disable_all);
+	static_call_update(x86_pmu_enable_all, x86_pmu.enable_all);
+	static_call_update(x86_pmu_enable, x86_pmu.enable);
+	static_call_update(x86_pmu_disable, x86_pmu.disable);
+
+	static_call_update(x86_pmu_add, x86_pmu.add);
+	static_call_update(x86_pmu_del, x86_pmu.del);
+	static_call_update(x86_pmu_read, x86_pmu.read);
+
+	static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events);
+	static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints);
+	static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints);
+
+	static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling);
+	static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling);
+	static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);
+
+	static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
+	static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);
+
+	static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
+	static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);
+}
+
+static void _x86_pmu_read(struct perf_event *event)
+{
+	x86_perf_event_update(event);
+}
+
 static int __init init_hw_perf_events(void)
 {
 	struct x86_pmu_quirk *quirk;
@@ -1885,6 +1934,11 @@ static int __init init_hw_perf_events(vo
 	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
 	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
 
+	if (!x86_pmu.read)
+		x86_pmu.read = _x86_pmu_read;
+
+	x86_pmu_static_call_update();
+
 	/*
 	 * Install callbacks. Core will call them for each online
 	 * cpu.
@@ -1921,11 +1975,9 @@ static int __init init_hw_perf_events(vo
 }
 early_initcall(init_hw_perf_events);
 
-static inline void x86_pmu_read(struct perf_event *event)
+static void x86_pmu_read(struct perf_event *event)
 {
-	if (x86_pmu.read)
-		return x86_pmu.read(event);
-	x86_perf_event_update(event);
+	static_call(x86_pmu_read)(event);
 }
 
 /*
@@ -2002,7 +2054,7 @@ static int x86_pmu_commit_txn(struct pmu
 	if (!x86_pmu_initialized())
 		return -EAGAIN;
 
-	ret = x86_pmu.schedule_events(cpuc, n, assign);
+	ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
 	if (ret)
 		return ret;
 
@@ -2300,15 +2352,13 @@ static const struct attribute_group *x86
 
 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
 {
-	if (x86_pmu.sched_task)
-		x86_pmu.sched_task(ctx, sched_in);
+	static_cond_call(x86_pmu_sched_task)(ctx, sched_in);
 }
 
 static void x86_pmu_swap_task_ctx(struct perf_event_context *prev,
 				  struct perf_event_context *next)
 {
-	if (x86_pmu.swap_task_ctx)
-		x86_pmu.swap_task_ctx(prev, next);
+	static_cond_call(x86_pmu_swap_task_ctx)(prev, next);
 }
 
 void perf_check_microcode(void)