Add static_call
Subsystems (15)
Peter Zijlstra
maintainerv7 cover Add static_call
Hi all,
static_call(), is the idea of static_branch() applied to indirect function calls. Remove a data load (indirection) by modifying the text.
The inline implementation still relies on objtool to generate the .static_call_sites section, mostly because this is a natural place for x86_64 and works for both GCC and LLVM. Other architectures can pick other means if/when they implement the inline patching. The out-of-line (aka. trampoline) variant doesn't require this.
Patches go on top of linus' tree.
Changes since last time:
- rebased
- review feedback from Steve
Peter Zijlstra
maintainerPATCH v7 1/18 notifier: Fix broken error handling pattern
The current notifiers have the following error handling pattern all over the place:
int err, nr; err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr); if (err & NOTIFIER_STOP_MASK) __foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL)
And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless.
Fix this by replacing all the __foo_notifier_call_chain() functions with foo_notifier_call_chain_robust() that embeds the above pattern, but ensures it is inside a single lock region.
Note: I switched atomic_notifier_call_chain_robust() to use
the spinlock, since RCU cannot provide the guarantee required for the recovery.
Note: software_resume() error handling was broken afaict.
--- include/linux/notifier.h | 15 +-- kernel/cpu_pm.c | 48 ++++-------- kernel/notifier.c | 144 ++++++++++++++++++++++--------------- kernel/power/hibernate.c | 39 ++++------ kernel/power/main.c | 8 +- kernel/power/power.h | 3 kernel/power/suspend.c | 14 +-- kernel/power/user.c | 14 +-- tools/power/pm-graph/sleepgraph.py | 2 9 files changed, 147 insertions(+), 140 deletions(-)
include/linux/notifier.h +7 -8
@@ -161,20 +161,19 @@ extern int srcu_notifier_chain_unregisteextern int atomic_notifier_call_chain (struct atomic_notifier_head *nh,unsigned long val,void *v); -extern int __atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int blocking_notifier_call_chain (struct blocking_notifier_head *nh,unsigned long val,void *v); -extern int __blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int raw_notifier_call_chain (struct raw_notifier_head *nh,unsigned long val,void *v); -extern int __raw_notifier_call_chain (struct raw_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int srcu_notifier_call_chain (struct srcu_notifier_head *nh,unsigned long val,void *v); -extern int __srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls); + +extern int atomic_notifier_call_chain_robust (struct atomic_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int blocking_notifier_call_chain_robust (struct blocking_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int raw_notifier_call_chain_robust (struct raw_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v);#define NOTIFY_DONE 0x0000/* Don't care */ #define NOTIFY_OK 0x0001/* Suits me */
kernel/cpu_pm.c +18 -30
86 lines not on this page · raw message
kernel/notifier.c +88 -56
240 lines not on this page · raw message
kernel/power/hibernate.c +18 -21
@@ -706,8 +706,8 @@ static int load_image_and_restore(void)*/ int hibernate (void ) { -int error, nr_calls =0 ;bool snapshot_test = false; +int error;if (!hibernation_available ()) {pm_pr_dbg ("Hibernation not available.\n" );
@@ -723,11 +723,9 @@ int hibernate(void)pr_info ("hibernation entry\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto Restore;ksys_sync_helper ();
@@ -785,7 +783,8 @@ int hibernate(void)/* Don't bother checking whether freezer_test_done is true */ freezer_test_done = false; Exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); + Restore:pm_restore_console ();hibernate_release (); Unlock:
@@ -804,7 +803,7 @@ int hibernate(void)*/ int hibernate_quiet_exec (int (*func)(void *data),void *data) { -int error, nr_calls =0 ; +int error;lock_system_sleep ();
@@ -815,11 +814,9 @@ int hibernate_quiet_exec(int (*func)(voipm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto restore; error =freeze_processes ();if (error)
@@ -880,8 +877,9 @@ int hibernate_quiet_exec(int (*func)(voithaw_processes (); exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); +restore:pm_restore_console ();hibernate_release ();
@@ -910,7 +908,7 @@ EXPORT_SYMBOL_GPL(hibernate_quiet_exec);*/ static int software_resume (void ) { -int error, nr_calls =0 ; +int error;/* * If the user said "noresume".. bail out early.
@@ -997,11 +995,9 @@ static int software_resume(void)pr_info ("resume from hibernation\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Close_Finish; - } + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE ); +if (error) + goto Restore;pm_pr_dbg ("Preparing processes for hibernation restore.\n" ); error =freeze_processes ();
@@ -1017,7 +1013,8 @@ static int software_resume(void) error =load_image_and_restore ();thaw_processes (); Finish: -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_RESTORE ); + Restore:pm_restore_console ();pr_info ("resume failed (%d)\n" , error);hibernate_release ();
kernel/power/main.c +4 -4
@@ -80,18 +80,18 @@ int unregister_pm_notifier(struct notifi }EXPORT_SYMBOL_GPL (unregister_pm_notifier); -int __pm_notifier_call_chain (unsigned long val,int nr_to_call,int *nr_calls) +int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down) {int ret; - ret =__blocking_notifier_call_chain (&pm_chain_head, val,NULL , - nr_to_call, nr_calls); + ret =blocking_notifier_call_chain_robust (&pm_chain_head, val_up, val_down,NULL );return notifier_to_errno (ret); } +int pm_notifier_call_chain (unsigned long val) { -return __pm_notifier_call_chain (val,-1 ,NULL ); +return blocking_notifier_call_chain (&pm_chain_head, val,NULL ); }/* If set, devices may be suspended and resumed asynchronously. */
kernel/power/power.h +1 -2
@@ -210,8 +210,7 @@ static inline void suspend_test_finish(c#ifdef CONFIG_PM_SLEEP /* kernel/power/main.c */ -extern int __pm_notifier_call_chain (unsigned long val,int nr_to_call, -int *nr_calls); +extern int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down);extern int pm_notifier_call_chain (unsigned long val);#endif
kernel/power/suspend.c +6 -8
@@ -342,18 +342,16 @@ static int suspend_test(int level)*/ static int suspend_prepare (suspend_state_t state) { -int error, nr_calls =0 ; +int error;if (!sleep_state_supported (state))return -EPERM ;pm_prepare_console (); - error =__pm_notifier_call_chain (PM_SUSPEND_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Finish; - } + error =pm_notifier_call_chain_robust (PM_SUSPEND_PREPARE ,PM_POST_SUSPEND ); +if (error) + goto Restore;trace_suspend_resume (TPS ("freeze_processes" ),0 , true); error =suspend_freeze_processes ();
@@ -363,8 +361,8 @@ static int suspend_prepare(suspend_state suspend_stats.failed_freeze++;dpm_save_failed_step (SUSPEND_FREEZE ); - Finish: -__pm_notifier_call_chain (PM_POST_SUSPEND , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_SUSPEND ); + Restore:pm_restore_console ();return error; }
kernel/power/user.c +4 -10
@@ -46,7 +46,7 @@ int is_hibernate_resume_dev(const structstatic int snapshot_open (struct inode *inode,struct file *filp) {struct snapshot_data *data; -int error, nr_calls =0 ; +int error;if (!hibernation_available ())return -EPERM ;
@@ -73,9 +73,7 @@ static int snapshot_open(struct inode *i swap_type_of(swsusp_resume_device, 0, NULL) : -1; data->mode =O_RDONLY ; data->free_bitmaps = false; - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) -__pm_notifier_call_chain (PM_POST_HIBERNATION , --nr_calls,NULL ); + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); }else {/* * Resuming. We may need to wait for the image device to
@@ -85,15 +83,11 @@ static int snapshot_open(struct inode *i data->swap =-1 ; data->mode =O_WRONLY ; - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE );if (!error) { error =create_basic_memory_bitmaps (); data->free_bitmaps = !error; - }else - nr_calls--; - -if (error) -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); + } }if (error)hibernate_release ();
tools/power/pm-graph/sleepgraph.py +1 -1
@@ -171,7 +171,7 @@ import base64 tracefuncs = { 'sys_sync': {}, 'ksys_sync': {}, - '__pm_notifier_call_chain': {}, + 'pm_notifier_call_chain_robust': {}, 'pm_prepare_console': {}, 'pm_notifier_call_chain': {}, 'freeze_processes': {},
[tip: core/static_call] notifier: Fix broken error handling pattern
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 70d932985757fbe978024db313001218e9f8fe5c Gitweb: https://git.kernel.org/tip/70d932985757fbe978024db313001218e9f8fe5c Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:36 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:03 +02:00
notifier: Fix broken error handling pattern
The current notifiers have the following error handling pattern all over the place:
int err, nr; err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr); if (err & NOTIFIER_STOP_MASK) __foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL)
And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless.
Fix this by replacing all the __foo_notifier_call_chain() functions with foo_notifier_call_chain_robust() that embeds the above pattern, but ensures it is inside a single lock region.
Note: I switched atomic_notifier_call_chain_robust() to use
the spinlock, since RCU cannot provide the guarantee required for the recovery.
Note: software_resume() error handling was broken afaict.
--- include/linux/notifier.h | 15 +-- kernel/cpu_pm.c | 48 +++------ kernel/notifier.c | 144 +++++++++++++++++----------- kernel/power/hibernate.c | 39 +++----- kernel/power/main.c | 8 +- kernel/power/power.h | 3 +- kernel/power/suspend.c | 14 +-- kernel/power/user.c | 14 +--- tools/power/pm-graph/sleepgraph.py | 2 +- 9 files changed, 147 insertions(+), 140 deletions(-)
include/linux/notifier.h +7 -8
@@ -161,20 +161,19 @@ extern int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,extern int atomic_notifier_call_chain (struct atomic_notifier_head *nh,unsigned long val,void *v); -extern int __atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int blocking_notifier_call_chain (struct blocking_notifier_head *nh,unsigned long val,void *v); -extern int __blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int raw_notifier_call_chain (struct raw_notifier_head *nh,unsigned long val,void *v); -extern int __raw_notifier_call_chain (struct raw_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int srcu_notifier_call_chain (struct srcu_notifier_head *nh,unsigned long val,void *v); -extern int __srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls); + +extern int atomic_notifier_call_chain_robust (struct atomic_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int blocking_notifier_call_chain_robust (struct blocking_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int raw_notifier_call_chain_robust (struct raw_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v);#define NOTIFY_DONE 0x0000/* Don't care */ #define NOTIFY_OK 0x0001/* Suits me */
kernel/cpu_pm.c +18 -30
86 lines not on this page · raw message
kernel/notifier.c +88 -56
240 lines not on this page · raw message
kernel/power/hibernate.c +18 -21
@@ -706,8 +706,8 @@ static int load_image_and_restore(void)*/ int hibernate (void ) { -int error, nr_calls =0 ;bool snapshot_test = false; +int error;if (!hibernation_available ()) {pm_pr_dbg ("Hibernation not available.\n" );
@@ -723,11 +723,9 @@ int hibernate(void)pr_info ("hibernation entry\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto Restore;ksys_sync_helper ();
@@ -785,7 +783,8 @@ int hibernate(void)/* Don't bother checking whether freezer_test_done is true */ freezer_test_done = false; Exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); + Restore:pm_restore_console ();hibernate_release (); Unlock:
@@ -804,7 +803,7 @@ int hibernate(void)*/ int hibernate_quiet_exec (int (*func)(void *data),void *data) { -int error, nr_calls =0 ; +int error;lock_system_sleep ();
@@ -815,11 +814,9 @@ int hibernate_quiet_exec(int (*func)(void *data), void *data)pm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto restore; error =freeze_processes ();if (error)
@@ -880,8 +877,9 @@ thaw:thaw_processes (); exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); +restore:pm_restore_console ();hibernate_release ();
@@ -910,7 +908,7 @@ EXPORT_SYMBOL_GPL(hibernate_quiet_exec);*/ static int software_resume (void ) { -int error, nr_calls =0 ; +int error;/* * If the user said "noresume".. bail out early.
@@ -997,11 +995,9 @@ static int software_resume(void)pr_info ("resume from hibernation\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Close_Finish; - } + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE ); +if (error) + goto Restore;pm_pr_dbg ("Preparing processes for hibernation restore.\n" ); error =freeze_processes ();
@@ -1017,7 +1013,8 @@ static int software_resume(void) error =load_image_and_restore ();thaw_processes (); Finish: -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_RESTORE ); + Restore:pm_restore_console ();pr_info ("resume failed (%d)\n" , error);hibernate_release ();
kernel/power/main.c +4 -4
@@ -80,18 +80,18 @@ int unregister_pm_notifier(struct notifier_block *nb) }EXPORT_SYMBOL_GPL (unregister_pm_notifier); -int __pm_notifier_call_chain (unsigned long val,int nr_to_call,int *nr_calls) +int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down) {int ret; - ret =__blocking_notifier_call_chain (&pm_chain_head, val,NULL , - nr_to_call, nr_calls); + ret =blocking_notifier_call_chain_robust (&pm_chain_head, val_up, val_down,NULL );return notifier_to_errno (ret); } +int pm_notifier_call_chain (unsigned long val) { -return __pm_notifier_call_chain (val,-1 ,NULL ); +return blocking_notifier_call_chain (&pm_chain_head, val,NULL ); }/* If set, devices may be suspended and resumed asynchronously. */
kernel/power/power.h +1 -2
@@ -210,8 +210,7 @@ static inline void suspend_test_finish(const char *label) {}#ifdef CONFIG_PM_SLEEP /* kernel/power/main.c */ -extern int __pm_notifier_call_chain (unsigned long val,int nr_to_call, -int *nr_calls); +extern int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down);extern int pm_notifier_call_chain (unsigned long val);#endif
kernel/power/suspend.c +6 -8
@@ -342,18 +342,16 @@ static int suspend_test(int level)*/ static int suspend_prepare (suspend_state_t state) { -int error, nr_calls =0 ; +int error;if (!sleep_state_supported (state))return -EPERM ;pm_prepare_console (); - error =__pm_notifier_call_chain (PM_SUSPEND_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Finish; - } + error =pm_notifier_call_chain_robust (PM_SUSPEND_PREPARE ,PM_POST_SUSPEND ); +if (error) + goto Restore;trace_suspend_resume (TPS ("freeze_processes" ),0 , true); error =suspend_freeze_processes ();
@@ -363,8 +361,8 @@ static int suspend_prepare(suspend_state_t state) suspend_stats.failed_freeze++;dpm_save_failed_step (SUSPEND_FREEZE ); - Finish: -__pm_notifier_call_chain (PM_POST_SUSPEND , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_SUSPEND ); + Restore:pm_restore_console ();return error; }
kernel/power/user.c +4 -10
@@ -46,7 +46,7 @@ int is_hibernate_resume_dev(const struct inode *bd_inode)static int snapshot_open (struct inode *inode,struct file *filp) {struct snapshot_data *data; -int error, nr_calls =0 ; +int error;if (!hibernation_available ())return -EPERM ;
@@ -73,9 +73,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) swap_type_of(swsusp_resume_device, 0, NULL) : -1; data->mode =O_RDONLY ; data->free_bitmaps = false; - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) -__pm_notifier_call_chain (PM_POST_HIBERNATION , --nr_calls,NULL ); + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); }else {/* * Resuming. We may need to wait for the image device to
@@ -85,15 +83,11 @@ static int snapshot_open(struct inode *inode, struct file *filp) data->swap =-1 ; data->mode =O_WRONLY ; - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE );if (!error) { error =create_basic_memory_bitmaps (); data->free_bitmaps = !error; - }else - nr_calls--; - -if (error) -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); + } }if (error)hibernate_release ();
tools/power/pm-graph/sleepgraph.py +1 -1
@@ -171,7 +171,7 @@ class SystemValues: tracefuncs = { 'sys_sync': {}, 'ksys_sync': {}, - '__pm_notifier_call_chain': {}, + 'pm_notifier_call_chain_robust': {}, 'pm_prepare_console': {}, 'pm_notifier_call_chain': {}, 'freeze_processes': {},
Peter Zijlstra
maintainerPATCH v7 2/18 module: Fix up module_notifier return values
In reply to Peter ZijlstraWhile auditing all module notifiers I noticed a whole bunch of fail wrt the return value. Notifiers have a 'special' return semantics.
As is; NOTIFY_DONE vs NOTIFY_OK is a bit vague; but notifier_from_errno(0) results in NOTIFY_OK and NOTIFY_DONE has a comment that says "Don't care".
Quoted 1 line
From this I've used
NOTIFY_DONEwhen the function completely ignores
the callback and notifier_to_error() isn't used.
--- drivers/oprofile/buffer_sync.c | 4 ++-- kernel/trace/bpf_trace.c | 8 ++++++-- kernel/trace/trace.c | 2 +- kernel/trace/trace_events.c | 2 +- kernel/trace/trace_printk.c | 4 ++-- kernel/tracepoint.c | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-)
drivers/oprofile/buffer_sync.c +2 -2
@@ -116,7 +116,7 @@ module_load_notify(struct notifier_block {#ifdef CONFIG_MODULES if (val !=MODULE_STATE_COMING ) -return 0 ; +return NOTIFY_DONE ;/* FIXME: should we process all CPU buffers ? */ mutex_lock (&buffer_mutex);
@@ -124,7 +124,7 @@ module_load_notify(struct notifier_blockadd_event_entry (MODULE_LOADED_CODE );mutex_unlock (&buffer_mutex);#endif -return 0 ; +return NOTIFY_OK ; }
kernel/trace/bpf_trace.c +6 -2
@@ -1451,10 +1451,11 @@ static int bpf_event_notify(struct notif {struct bpf_trace_module *btm, *tmp;struct module *mod = module; +int ret =0 ;if (mod->num_bpf_raw_events ==0 || (op !=MODULE_STATE_COMING && op !=MODULE_STATE_GOING )) -return 0 ; + goto out;mutex_lock (&bpf_module_mutex);
@@ -1464,6 +1465,8 @@ static int bpf_event_notify(struct notifif (btm) { btm->module = module;list_add (&btm->list, &bpf_trace_modules); + }else { + ret = -ENOMEM ; }break ;case MODULE_STATE_GOING :
@@ -1479,7 +1482,8 @@ static int bpf_event_notify(struct notifmutex_unlock (&bpf_module_mutex); -return 0 ; +out: +return notifier_from_errno (ret); }static struct notifier_block bpf_module_nb = {
kernel/trace/trace.c +1 -1
@@ -8696,7 +8696,7 @@ static int trace_module_notify(struct nobreak ; } -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_events.c +1 -1
@@ -2442,7 +2442,7 @@ static int trace_module_notify(struct nomutex_unlock (&trace_types_lock);mutex_unlock (&event_mutex); -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_printk.c +2 -2
@@ -95,7 +95,7 @@ static int module_trace_bprintk_format_nif (val ==MODULE_STATE_COMING )hold_module_trace_bprintk_format (start, end); } -return 0 ; +return NOTIFY_OK ; }/*
@@ -173,7 +173,7 @@ __init static int module_trace_bprintk_format_notify(struct notifier_block *self, unsigned long val, void *data) { - return 0; + return NOTIFY_OK; }static inline const char **find_next_mod_format (int start_index,void *v,const char **fmt,loff_t *pos)
kernel/tracepoint.c +1 -1
@@ -521,7 +521,7 @@ static int tracepoint_module_notify(strucase MODULE_STATE_UNFORMED :break ; } -return ret; +return notifier_from_errno (ret); }static struct notifier_block tracepoint_module_nb = {
[tip: core/static_call] module: Fix up module_notifier return values
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 0340a6b7fb767f7f296b9bacc9a215920519a644 Gitweb: https://git.kernel.org/tip/0340a6b7fb767f7f296b9bacc9a215920519a644 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:37 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:03 +02:00
module: Fix up module_notifier return values
While auditing all module notifiers I noticed a whole bunch of fail wrt the return value. Notifiers have a 'special' return semantics.
As is; NOTIFY_DONE vs NOTIFY_OK is a bit vague; but notifier_from_errno(0) results in NOTIFY_OK and NOTIFY_DONE has a comment that says "Don't care".
Quoted 1 line
From this I've used
NOTIFY_DONEwhen the function completely ignores
the callback and notifier_to_error() isn't used.
--- drivers/oprofile/buffer_sync.c | 4 ++-- kernel/trace/bpf_trace.c | 8 ++++++-- kernel/trace/trace.c | 2 +- kernel/trace/trace_events.c | 2 +- kernel/trace/trace_printk.c | 4 ++-- kernel/tracepoint.c | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-)
drivers/oprofile/buffer_sync.c +2 -2
@@ -116,7 +116,7 @@ module_load_notify(struct notifier_block *self, unsigned long val, void *data) {#ifdef CONFIG_MODULES if (val !=MODULE_STATE_COMING ) -return 0 ; +return NOTIFY_DONE ;/* FIXME: should we process all CPU buffers ? */ mutex_lock (&buffer_mutex);
@@ -124,7 +124,7 @@ module_load_notify(struct notifier_block *self, unsigned long val, void *data)add_event_entry (MODULE_LOADED_CODE );mutex_unlock (&buffer_mutex);#endif -return 0 ; +return NOTIFY_OK ; }
kernel/trace/bpf_trace.c +6 -2
@@ -2027,10 +2027,11 @@ static int bpf_event_notify(struct notifier_block *nb, unsigned long op, {struct bpf_trace_module *btm, *tmp;struct module *mod = module; +int ret =0 ;if (mod->num_bpf_raw_events ==0 || (op !=MODULE_STATE_COMING && op !=MODULE_STATE_GOING )) -return 0 ; + goto out;mutex_lock (&bpf_module_mutex);
@@ -2040,6 +2041,8 @@ static int bpf_event_notify(struct notifier_block *nb, unsigned long op,if (btm) { btm->module = module;list_add (&btm->list, &bpf_trace_modules); + }else { + ret = -ENOMEM ; }break ;case MODULE_STATE_GOING :
@@ -2055,7 +2058,8 @@ static int bpf_event_notify(struct notifier_block *nb, unsigned long op,mutex_unlock (&bpf_module_mutex); -return 0 ; +out: +return notifier_from_errno (ret); }static struct notifier_block bpf_module_nb = {
kernel/trace/trace.c +1 -1
@@ -9072,7 +9072,7 @@ static int trace_module_notify(struct notifier_block *self,break ; } -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_events.c +1 -1
@@ -2646,7 +2646,7 @@ static int trace_module_notify(struct notifier_block *self,mutex_unlock (&trace_types_lock);mutex_unlock (&event_mutex); -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_printk.c +2 -2
@@ -96,7 +96,7 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self,if (val ==MODULE_STATE_COMING )hold_module_trace_bprintk_format (start, end); } -return 0 ; +return NOTIFY_OK ; }/*
@@ -174,7 +174,7 @@ __init static int module_trace_bprintk_format_notify(struct notifier_block *self, unsigned long val, void *data) { - return 0; + return NOTIFY_OK; }static inline const char **find_next_mod_format (int start_index,void *v,const char **fmt,loff_t *pos)
kernel/tracepoint.c +1 -1
@@ -521,7 +521,7 @@ static int tracepoint_module_notify(struct notifier_block *self,case MODULE_STATE_UNFORMED :break ; } -return ret; +return notifier_from_errno (ret); }static struct notifier_block tracepoint_module_nb = {
Peter Zijlstra
maintainerPATCH v7 3/18 module: Properly propagate MODULE_STATE_COMING failure
In reply to Peter ZijlstraNow that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.
There were already MODULE_STATE_COMING notifiers that failed; notably:
- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()
By propagating this error, we fix those users.
--- kernel/module.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +7 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(structif (err)return err; -blocking_notifier_call_chain (&module_notify_list, -MODULE_STATE_COMING , mod); -return 0 ; + err =blocking_notifier_call_chain_robust (&module_notify_list, +MODULE_STATE_COMING ,MODULE_STATE_GOING , mod); + err =notifier_to_errno (err); +if (err) +klp_module_going (mod); + +return err; } static int unknown_module_param_cb(char *param, char *val, const char *modname,
[tip: core/static_call] module: Properly propagate MODULE_STATE_COMING failure
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 59cc8e0a906ea23190922e5e0252e5b5a60d70c2 Gitweb: https://git.kernel.org/tip/59cc8e0a906ea23190922e5e0252e5b5a60d70c2 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:38 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
module: Properly propagate MODULE_STATE_COMING failure
Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.
There were already MODULE_STATE_COMING notifiers that failed; notably:
- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()
By propagating this error, we fix those users.
--- kernel/module.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +7 -3
@@ -3792,9 +3792,13 @@ static int prepare_coming_module(struct module *mod)if (err)return err; -blocking_notifier_call_chain (&module_notify_list, -MODULE_STATE_COMING , mod); -return 0 ; + err =blocking_notifier_call_chain_robust (&module_notify_list, +MODULE_STATE_COMING ,MODULE_STATE_GOING , mod); + err =notifier_to_errno (err); +if (err) +klp_module_going (mod); + +return err; } static int unknown_module_param_cb(char *param, char *val, const char *modname,
Peter Zijlstra
maintainerPATCH v7 4/18 jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved
In reply to Peter ZijlstraNothing ensures the module exists while we're iterating mod->jump_entries in __jump_label_mod_text_reserved(), take a module reference to ensure the module sticks around.
--- kernel/jump_label.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/jump_label.c +8 -2
@@ -539,19 +539,25 @@ static void static_key_set_mod(struct ststatic int __jump_label_mod_text_reserved (void *start,void *end) {struct module *mod; +int ret;preempt_disable (); mod =__module_text_address ((unsigned long )start);WARN_ON_ONCE (__module_text_address ((unsigned long )end) != mod); +if (!try_module_get (mod)) + mod =NULL ;preempt_enable ();if (!mod)return 0 ; - -return __jump_label_text_reserved (mod->jump_entries, + ret =__jump_label_text_reserved (mod->jump_entries, mod->jump_entries + mod->num_jump_entries, start, end); + +module_put (mod); + +return ret; }static void __jump_label_mod_update (struct static_key *key)
[tip: core/static_call] jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved()
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 0db6e3734b130207026df1e78455fa98ca1d6f50 Gitweb: https://git.kernel.org/tip/0db6e3734b130207026df1e78455fa98ca1d6f50 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:39 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved()
Nothing ensures the module exists while we're iterating mod->jump_entries in __jump_label_mod_text_reserved(), take a module reference to ensure the module sticks around.
--- kernel/jump_label.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/jump_label.c +8 -2
@@ -539,19 +539,25 @@ static void static_key_set_mod(struct static_key *key,static int __jump_label_mod_text_reserved (void *start,void *end) {struct module *mod; +int ret;preempt_disable (); mod =__module_text_address ((unsigned long )start);WARN_ON_ONCE (__module_text_address ((unsigned long )end) != mod); +if (!try_module_get (mod)) + mod =NULL ;preempt_enable ();if (!mod)return 0 ; - -return __jump_label_text_reserved (mod->jump_entries, + ret =__jump_label_text_reserved (mod->jump_entries, mod->jump_entries + mod->num_jump_entries, start, end); + +module_put (mod); + +return ret; }static void __jump_label_mod_update (struct static_key *key)
Peter Zijlstra
maintainerPATCH v7 5/18 compiler.h: Make __ADDRESSABLE() symbol truly unique
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.
Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.
--- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -299,7 +299,7 @@ unsigned long read_word_at_a_time(const*/ #define __ADDRESSABLE (sym) \ static void * __section(.discard.addressable) __used \ - __PASTE(__addressable_##sym, __LINE__) = (void *)&sym; + __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;/** * offset_to_ptr - convert a relative memory offset to an absolute pointer
[tip: core/static_call] compiler.h: Make __ADDRESSABLE() symbol truly unique
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 563a02b0c9704f69c0364befedd451f57fe88092 Gitweb: https://git.kernel.org/tip/563a02b0c9704f69c0364befedd451f57fe88092 Author: Josh Poimboeuf <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:40 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
compiler.h: Make __ADDRESSABLE() symbol truly unique
The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.
Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.
--- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -207,7 +207,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,*/ #define __ADDRESSABLE (sym) \ static void * __section(.discard.addressable) __used \ - __PASTE(__addressable_##sym, __LINE__) = (void *)&sym; + __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;/** * offset_to_ptr - convert a relative memory offset to an absolute pointer
Peter Zijlstra
maintainerPATCH v7 6/18 static_call: Add basic static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:
https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]
There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 3 include/linux/static_call.h | 156 ++++++++++++++++++++++++++++++++++++++ include/linux/static_call_types.h | 15 +++ 3 files changed, 174 insertions(+) create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -975,6 +975,9 @@ config HAVE_SPARSE_SYSCALL_NR config ARCH_HAS_VDSO_DATA bool +config HAVE_STATIC_CALL + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +156 -0
156 lines not on this page · raw message
include/linux/static_call_types.h new file +15 -0
@@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name) + +#define STATIC_CALL_TRAMP_PREFIX __SCT__ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +#endif /* _STATIC_CALL_TYPES_H */
[tip: core/static_call] static_call: Add basic static call infrastructure
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 115284d89a436e9b66da0c6c4f6efded806874b2 Gitweb: https://git.kernel.org/tip/115284d89a436e9b66da0c6c4f6efded806874b2 Author: Josh Poimboeuf <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:41 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
static_call: Add basic static call infrastructure
Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:
https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]
There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers
For more details, see the comments in include/linux/static_call.h.
[peterz: simplified interface]
--- arch/Kconfig | 3 +- include/linux/static_call.h | 156 +++++++++++++++++++++++++++++- include/linux/static_call_types.h | 15 +++- 3 files changed, 174 insertions(+) create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -975,6 +975,9 @@ config HAVE_SPARSE_SYSCALL_NR config ARCH_HAS_VDSO_DATA bool +config HAVE_STATIC_CALL + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +156 -0
156 lines not on this page · raw message
include/linux/static_call_types.h new file +15 -0
@@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name) + +#define STATIC_CALL_TRAMP_PREFIX __SCT__ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +#endif /* _STATIC_CALL_TYPES_H */
Peter Zijlstra
maintainerPATCH v7 7/18 static_call: Add inline static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.
Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:
https://lkml.kernel.org/r/[email protected]
This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 4 include/asm-generic/vmlinux.lds.h | 7 include/linux/module.h | 5 include/linux/static_call.h | 36 ++++ include/linux/static_call_types.h | 13 + kernel/Makefile | 1 kernel/module.c | 5 kernel/static_call.c | 303 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -982,6 +982,10 @@ config HAVE_SPARSE_SYSCALL_NR config HAVE_STATIC_CALL bool +config HAVE_STATIC_CALL_INLINE + bool + depends on HAVE_STATIC_CALL + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -368,6 +368,12 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#define STATIC_CALL_DATA \ + . = ALIGN(8); \ + __start_static_call_sites = .; \ + KEEP(*(.static_call_sites)) \ + __stop_static_call_sites = .; +/* * Allow architectures to handle ro_after_init data on their * own by defining an empty RO_AFTER_INIT_DATA.
@@ -377,6 +383,7 @@ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ + STATIC_CALL_DATA \ __end_ro_after_init = .;#endif
include/linux/module.h +5 -0
@@ -25,6 +25,7 @@#include <linux/error-injection.h> #include <linux/tracepoint-defs.h> #include <linux/srcu.h> +#include <linux/static_call_types.h> #include <linux/percpu.h> #include <asm/module.h>
@@ -497,6 +498,10 @@ struct module {unsigned long *kprobe_blacklist;unsigned int num_kprobe_blacklist;#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +int num_static_call_sites; +struct static_call_site *static_call_sites; +#endif #ifdef CONFIG_LIVEPATCH bool klp;/* Is this a livepatch module? */
include/linux/static_call.h +35 -1
@@ -95,7 +95,41 @@ extern void arch_static_call_transform(v STATIC_CALL_TRAMP_ADDR(name), func); \ }) -#if defined(CONFIG_HAVE_STATIC_CALL) +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + +struct static_call_mod { + struct static_call_mod *next; + struct module *mod; /* for vmlinux, mod == NULL */ + struct static_call_site *sites; +}; + +struct static_call_key { + void *func; + struct static_call_mod *mods; +}; + +extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); +extern int static_call_mod_init(struct module *mod); + +#define DEFINE_STATIC_CALL(name, _func) \ + DECLARE_STATIC_CALL(name, _func); \ + struct static_call_key STATIC_CALL_KEY(name) = { \ + .func = _func, \ + .mods = NULL, \ + }; \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) + +#define static_call(name) __static_call(name) + +#define EXPORT_STATIC_CALL(name) \ + EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) + +#define EXPORT_STATIC_CALL_GPL(name) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) + +#elif defined(CONFIG_HAVE_STATIC_CALL) struct static_call_key { void *func;
include/linux/static_call_types.h +13 -0
@@ -2,14 +2,27 @@#ifndef _STATIC_CALL_TYPES_H#define _STATIC_CALL_TYPES_H +#include <linux/types.h> #include <linux/stringify.h> #define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) +#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name)#define STATIC_CALL_TRAMP_PREFIX __SCT__#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1)#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name)#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; +#endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -109,6 +109,7 @@ obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ obj-$(CONFIG_KCSAN) += kcsan/ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o +obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/module.c +5 -0
@@ -3226,6 +3226,11 @@ static int find_module_sections(struct msizeof (unsigned long ), &mod->num_kprobe_blacklist);#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + mod->static_call_sites =section_objs (info,".static_call_sites" , +sizeof (*mod->static_call_sites), + &mod->num_static_call_sites); +#endif mod->extable =section_objs (info,"__ex_table" ,sizeof (*mod->extable), &mod->num_exentries);
kernel/static_call.c new file +303 -0
303 lines not on this page · raw message
[tip: core/static_call] static_call: Add inline static call infrastructure
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 9183c3f9ed710a8edf1a61e8a96d497258d26e08 Gitweb: https://git.kernel.org/tip/9183c3f9ed710a8edf1a61e8a96d497258d26e08 Author: Josh Poimboeuf <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:42 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
static_call: Add inline static call infrastructure
Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.
Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:
https://lkml.kernel.org/r/[email protected]
This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.
For more details, see the comments in include/linux/static_call.h.
[peterz: simplified interface; merged trampolines]
--- arch/Kconfig | 4 +- include/asm-generic/vmlinux.lds.h | 7 +- include/linux/module.h | 5 +- include/linux/static_call.h | 36 ++- include/linux/static_call_types.h | 13 +- kernel/Makefile | 1 +- kernel/module.c | 5 +- kernel/static_call.c | 303 +++++++++++++++++++++++++++++- 8 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -978,6 +978,10 @@ config ARCH_HAS_VDSO_DATA config HAVE_STATIC_CALL bool +config HAVE_STATIC_CALL_INLINE + bool + depends on HAVE_STATIC_CALL + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -388,6 +388,12 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#define STATIC_CALL_DATA \ + . = ALIGN(8); \ + __start_static_call_sites = .; \ + KEEP(*(.static_call_sites)) \ + __stop_static_call_sites = .; +/* * Allow architectures to handle ro_after_init data on their * own by defining an empty RO_AFTER_INIT_DATA.
@@ -398,6 +404,7 @@ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ + STATIC_CALL_DATA \ __end_ro_after_init = .;#endif
include/linux/module.h +5 -0
@@ -25,6 +25,7 @@#include <linux/error-injection.h> #include <linux/tracepoint-defs.h> #include <linux/srcu.h> +#include <linux/static_call_types.h> #include <linux/percpu.h> #include <asm/module.h>
@@ -498,6 +499,10 @@ struct module {unsigned long *kprobe_blacklist;unsigned int num_kprobe_blacklist;#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +int num_static_call_sites; +struct static_call_site *static_call_sites; +#endif #ifdef CONFIG_LIVEPATCH bool klp;/* Is this a livepatch module? */
include/linux/static_call.h +35 -1
@@ -95,7 +95,41 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func); STATIC_CALL_TRAMP_ADDR(name), func); \ }) -#if defined(CONFIG_HAVE_STATIC_CALL) +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + +struct static_call_mod { + struct static_call_mod *next; + struct module *mod; /* for vmlinux, mod == NULL */ + struct static_call_site *sites; +}; + +struct static_call_key { + void *func; + struct static_call_mod *mods; +}; + +extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); +extern int static_call_mod_init(struct module *mod); + +#define DEFINE_STATIC_CALL(name, _func) \ + DECLARE_STATIC_CALL(name, _func); \ + struct static_call_key STATIC_CALL_KEY(name) = { \ + .func = _func, \ + .mods = NULL, \ + }; \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) + +#define static_call(name) __static_call(name) + +#define EXPORT_STATIC_CALL(name) \ + EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) + +#define EXPORT_STATIC_CALL_GPL(name) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) + +#elif defined(CONFIG_HAVE_STATIC_CALL) struct static_call_key { void *func;
include/linux/static_call_types.h +13 -0
@@ -2,14 +2,27 @@#ifndef _STATIC_CALL_TYPES_H#define _STATIC_CALL_TYPES_H +#include <linux/types.h> #include <linux/stringify.h> #define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) +#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name)#define STATIC_CALL_TRAMP_PREFIX __SCT__#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1)#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name)#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; +#endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -111,6 +111,7 @@ obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ obj-$(CONFIG_KCSAN) += kcsan/ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o +obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/module.c +5 -0
@@ -3275,6 +3275,11 @@ static int find_module_sections(struct module *mod, struct load_info *info)sizeof (unsigned long ), &mod->num_kprobe_blacklist);#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + mod->static_call_sites =section_objs (info,".static_call_sites" , +sizeof (*mod->static_call_sites), + &mod->num_static_call_sites); +#endif mod->extable =section_objs (info,"__ex_table" ,sizeof (*mod->extable), &mod->num_exentries);
kernel/static_call.c new file +303 -0
303 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v7 8/18 static_call: Avoid kprobes on inline static_call()s
In reply to Peter ZijlstraSimilar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
--- arch/x86/kernel/kprobes/opt.c | 4 +- include/linux/static_call.h | 11 ++++++ kernel/kprobes.c | 2 + kernel/static_call.c | 68 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 1 deletion(-)
arch/x86/kernel/kprobes/opt.c +3 -1
@@ -18,6 +18,7 @@#include <linux/ftrace.h> #include <linux/frame.h> #include <linux/pgtable.h> +#include <linux/static_call.h> #include <asm/text-patching.h> #include <asm/cacheflush.h>
@@ -210,7 +211,8 @@ static int copy_optimized_instructions(u/* Check whether the address range is reserved */ if (ftrace_text_reserved (src, src + len -1 ) ||alternatives_text_reserved (src, src + len -1 ) || -jump_label_text_reserved (src, src + len -1 )) +jump_label_text_reserved (src, src + len -1 ) || +static_call_text_reserved (src, src + len -1 ))return -EBUSY ;return len;
include/linux/static_call.h +11 -0
@@ -110,6 +110,7 @@ struct static_call_key {extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);extern int static_call_mod_init (struct module *mod); +extern int static_call_text_reserved (void *start,void *end);#define DEFINE_STATIC_CALL (name, _func) \ DECLARE_STATIC_CALL(name, _func); \
@@ -153,6 +154,11 @@ void __static_call_update(struct static_cpus_read_unlock (); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
@@ -182,6 +188,11 @@ void __static_call_update(struct static_WRITE_ONCE (key->func, func); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))#define EXPORT_STATIC_CALL_GPL (name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
kernel/kprobes.c +2 -0
@@ -36,6 +36,7 @@#include <linux/cpu.h> #include <linux/jump_label.h> #include <linux/perf_event.h> +#include <linux/static_call.h> #include <asm/sections.h> #include <asm/cacheflush.h>
@@ -1634,6 +1635,7 @@ static int check_kprobe_address_safe(strif (!kernel_text_address ((unsigned long ) p->addr) ||within_kprobe_blacklist ((unsigned long ) p->addr) ||jump_label_text_reserved (p->addr, p->addr) || +static_call_text_reserved (p->addr, p->addr) ||find_bug ((unsigned long )p->addr)) { ret = -EINVAL ; goto out;
kernel/static_call.c +68 -0
84 lines not on this page · raw message
[tip: core/static_call] static_call: Avoid kprobes on inline static_call()s
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 6333e8f73b834f54e395a056e6002403f0862c51 Gitweb: https://git.kernel.org/tip/6333e8f73b834f54e395a056e6002403f0862c51 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:43 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:04 +02:00
static_call: Avoid kprobes on inline static_call()s
Similar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
--- arch/x86/kernel/kprobes/opt.c | 4 +- include/linux/static_call.h | 11 ++++++- kernel/kprobes.c | 2 +- kernel/static_call.c | 68 ++++++++++++++++++++++++++++++++++- 4 files changed, 84 insertions(+), 1 deletion(-)
arch/x86/kernel/kprobes/opt.c +3 -1
@@ -18,6 +18,7 @@#include <linux/ftrace.h> #include <linux/frame.h> #include <linux/pgtable.h> +#include <linux/static_call.h> #include <asm/text-patching.h> #include <asm/cacheflush.h>
@@ -210,7 +211,8 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real)/* Check whether the address range is reserved */ if (ftrace_text_reserved (src, src + len -1 ) ||alternatives_text_reserved (src, src + len -1 ) || -jump_label_text_reserved (src, src + len -1 )) +jump_label_text_reserved (src, src + len -1 ) || +static_call_text_reserved (src, src + len -1 ))return -EBUSY ;return len;
include/linux/static_call.h +11 -0
@@ -110,6 +110,7 @@ struct static_call_key {extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);extern int static_call_mod_init (struct module *mod); +extern int static_call_text_reserved (void *start,void *end);#define DEFINE_STATIC_CALL (name, _func) \ DECLARE_STATIC_CALL(name, _func); \
@@ -153,6 +154,11 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)cpus_read_unlock (); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
@@ -182,6 +188,11 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)WRITE_ONCE (key->func, func); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))#define EXPORT_STATIC_CALL_GPL (name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
kernel/kprobes.c +2 -0
@@ -36,6 +36,7 @@#include <linux/cpu.h> #include <linux/jump_label.h> #include <linux/perf_event.h> +#include <linux/static_call.h> #include <asm/sections.h> #include <asm/cacheflush.h>
@@ -1634,6 +1635,7 @@ static int check_kprobe_address_safe(struct kprobe *p,if (!kernel_text_address ((unsigned long ) p->addr) ||within_kprobe_blacklist ((unsigned long ) p->addr) ||jump_label_text_reserved (p->addr, p->addr) || +static_call_text_reserved (p->addr, p->addr) ||find_bug ((unsigned long )p->addr)) { ret = -EINVAL ; goto out;
kernel/static_call.c +68 -0
84 lines not on this page · raw message
Re: [PATCH v7 08/18] static_call: Avoid kprobes on inline static_call()s
In reply to Peter ZijlstraQuoted 2 lines · On Tue, 18 Aug 2020 15:57:43 +0200 Peter Zijlstra <[email protected]> wrote:
Similar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
Looks good to me.
BTW, here we already have 5 subsystems which reserves texts (ftrace, alternatives, jump_label, static_call and kprobes.)
Except for the kprobes and ftrace, we can generalize the reserved-text code because those are section-based static address-areas (or lists).
Thank you,
Quoted 171 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kernel/kprobes/opt.c | 4 +- include/linux/static_call.h | 11 ++++++ kernel/kprobes.c | 2 + kernel/static_call.c | 68 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 1 deletion(-)arch/x86/kernel/kprobes/opt.c +3
-1@@ -18,6 +18,7 @@#include <linux/ftrace.h> #include <linux/frame.h> #include <linux/pgtable.h> +#include <linux/static_call.h> #include <asm/text-patching.h> #include <asm/cacheflush.h> @@ -210,7 +211,8 @@ static int copy_optimized_instructions(u/* Check whether the address range is reserved */ if (ftrace_text_reserved (src, src + len -1 ) ||alternatives_text_reserved (src, src + len -1 ) || -jump_label_text_reserved (src, src + len -1 )) +jump_label_text_reserved (src, src + len -1 ) || +static_call_text_reserved (src, src + len -1 ))return -EBUSY ;return len;include/linux/static_call.h +11
-0@@ -110,6 +110,7 @@ struct static_call_key {extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);extern int static_call_mod_init (struct module *mod); +extern int static_call_text_reserved (void *start,void *end);#define DEFINE_STATIC_CALL (name, _func) \ DECLARE_STATIC_CALL(name, _func); \@@ -153,6 +154,11 @@ void __static_call_update(struct static_cpus_read_unlock (); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))@@ -182,6 +188,11 @@ void __static_call_update(struct static_WRITE_ONCE (key->func, func); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))#define EXPORT_STATIC_CALL_GPL (name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))kernel/kprobes.c +2
-0@@ -36,6 +36,7 @@#include <linux/cpu.h> #include <linux/jump_label.h> #include <linux/perf_event.h> +#include <linux/static_call.h> #include <asm/sections.h> #include <asm/cacheflush.h> @@ -1634,6 +1635,7 @@ static int check_kprobe_address_safe(strif (!kernel_text_address ((unsigned long ) p->addr) ||within_kprobe_blacklist ((unsigned long ) p->addr) ||jump_label_text_reserved (p->addr, p->addr) || +static_call_text_reserved (p->addr, p->addr) ||find_bug ((unsigned long )p->addr)) { ret = -EINVAL ; goto out;kernel/static_call.c +68
-0@@ -204,8 +204,58 @@ static int __static_call_init(struct modreturn 0 ; } +static int addr_conflict (struct static_call_site *site,void *start,void *end) +{ +unsigned long addr = (unsigned long )static_call_addr (site); + +if (addr <= (unsigned long )end && + addr +CALL_INSN_SIZE > (unsigned long )start) +return 1 ; + +return 0 ; +} + +static int __static_call_text_reserved (struct static_call_site *iter_start, +struct static_call_site *iter_stop, +void *start,void *end) +{ +struct static_call_site *iter = iter_start; + +while (iter < iter_stop) { +if (addr_conflict (iter, start, end)) +return 1 ; + iter++; + } + +return 0 ; +} +#ifdef CONFIG_MODULES +static int __static_call_mod_text_reserved (void *start,void *end) +{ +struct module *mod; +int ret; + +preempt_disable (); + mod =__module_text_address ((unsigned long )start); +WARN_ON_ONCE (__module_text_address ((unsigned long )end) != mod); +if (!try_module_get (mod)) + mod =NULL ; +preempt_enable (); + +if (!mod) +return 0 ; + + ret =__static_call_text_reserved (mod->static_call_sites, + mod->static_call_sites + mod->num_static_call_sites, + start, end); + +module_put (mod); + +return ret; +} +static int static_call_add_module (struct module *mod) { return __static_call_init(mod, mod->static_call_sites,@@ -273,8 +323,26 @@ static struct notifier_block static_call .notifier_call = static_call_module_notify, }; +#else + +static inline int __static_call_mod_text_reserved (void *start,void *end) +{ +return 0 ; +} +#endif /* CONFIG_MODULES */ +int static_call_text_reserved (void *start,void *end) +{ +int ret =__static_call_text_reserved (__start_static_call_sites, + __stop_static_call_sites, start, end); + +if (ret) +return ret; + +return __static_call_mod_text_reserved (start, end); +} +static void __init static_call_init (void ) {int ret;
Signature · 1 line
Masami Hiramatsu <[email protected]>
[email protected]
maintainerRe: [PATCH v7 08/18] static_call: Avoid kprobes on inline static_call()s
Quoted 8 lines · Masami Hiramatsu
Quoted 2 lines · On Tue, 18 Aug 2020 15:57:43 +0200 Peter Zijlstra <[email protected]> wrote:
Similar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
Looks good to me.
Quoted 7 lines
Acked-by Masami Hiramatsu <[email protected]>BTW, here we already have 5 subsystems which reserves texts (ftrace, alternatives,
jump_label,static_calland kprobes.)Except for the kprobes and ftrace, we can generalize the reserved-text code because those are section-based static address-areas (or lists).
Doesn't ftrace also have a section where it lists all the mcount locations?
On top of that ftrace probably registers its trampolines.
Do we support adding kprobes to BPF-JIT'ed code or should we blacklist them too?
Masami Hiramatsu
maintainerRe: [PATCH v7 08/18] static_call: Avoid kprobes on inline static_call()s
Quoted 11 lines · On Wed, 2 Sep 2020 11:48:20 +0200 [email protected] wrote:
Quoted 10 lines · Masami Hiramatsu
Quoted 2 lines · On Tue, 18 Aug 2020 15:57:43 +0200 Peter Zijlstra <[email protected]> wrote:
Similar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
Looks good to me.
Acked-by Masami Hiramatsu <[email protected]>
Quoted 8 lines
Quoted 5 lines
BTW, here we already have 5 subsystems which reserves texts (ftrace, alternatives,
jump_label,static_calland kprobes.)Except for the kprobes and ftrace, we can generalize the reserved-text code because those are section-based static address-areas (or lists).
Doesn't ftrace also have a section where it lists all the mcount locations?
Is the data format in the section same as others?
Quoted 2 lines
On top of that ftrace probably registers its trampolines.
Good point. I think trampolines are filtered by kernel_text() check.
Quoted 3 lines
Do we support adding kprobes to BPF-JIT'ed code or should we blacklist them too?
No, we should blacklist it, because JIT'ed code will be used in kprobes context. In that case, I think it is better to provide partially execute non-JIT code and interrupt the interpreter.
Thank you,
Signature · 1 line
Masami Hiramatsu <[email protected]>
[email protected]
maintainerRe: [PATCH v7 08/18] static_call: Avoid kprobes on inline static_call()s
Quoted 1 line · Masami Hiramatsu
Is the data format in the section same as others?
All 3 sections (mcount, jump_label and static_call) have different layout.
Peter Zijlstra
maintainerPATCH v7 9/18 x86/static_call: Add out-of-line static call implementation
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/static_call.h | 23 +++++++++++++++++++++++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/static_call.c | 31 +++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -213,6 +213,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +23 -0
@@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +#include <asm/text-patching.h> + +/* +* For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which +* does a direct jump to the function. The direct jump gets patched by +* static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + " .byte 0xe9 # jmp.d32 \n" \ + " .long " #func " - (. + 4) \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -68,6 +68,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rt obj-y += pci-iommu_table.o obj-y += resource.o obj-y += irqflags.o +obj-y += static_call.o obj-y += process.o obj-y += fpu/
arch/x86/kernel/static_call.c new file +31 -0
@@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> + +static void __static_call_transform (void *insn,u8 opcode,void *func) +{ +const void *code =text_gen_insn (opcode, insn, func); + +if (WARN_ONCE (*(u8 *)insn != opcode, +"unexpected static call insn opcode 0x%x at %pS\n" , + opcode, insn)) +return ; + +if (memcmp (insn, code,CALL_INSN_SIZE ) ==0 ) +return ; + +text_poke_bp (insn, code,CALL_INSN_SIZE ,NULL ); +} + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +mutex_lock (&text_mutex); + +if (tramp) +__static_call_transform (tramp,JMP32_INSN_OPCODE , func); + +mutex_unlock (&text_mutex); +} +EXPORT_SYMBOL_GPL (arch_static_call_transform);
[tip: core/static_call] x86/static_call: Add out-of-line static call implementation
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: e6d6c071f22de29e4993784fc00cd2202b7ba149 Gitweb: https://git.kernel.org/tip/e6d6c071f22de29e4993784fc00cd2202b7ba149 Author: Josh Poimboeuf <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:44 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:05 +02:00
x86/static_call: Add out-of-line static call implementation
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.
[peterz: fixed trampoline, rewrote patching code]
--- arch/x86/Kconfig | 1 +- arch/x86/include/asm/static_call.h | 23 ++++++++++++++++++++++- arch/x86/kernel/Makefile | 1 +- arch/x86/kernel/static_call.c | 31 +++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+) create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -215,6 +215,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +23 -0
@@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +#include <asm/text-patching.h> + +/* +* For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which +* does a direct jump to the function. The direct jump gets patched by +* static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + " .byte 0xe9 # jmp.d32 \n" \ + " .long " #func " - (. + 4) \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -68,6 +68,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rtc.o obj-y += pci-iommu_table.o obj-y += resource.o obj-y += irqflags.o +obj-y += static_call.o obj-y += process.o obj-y += fpu/
arch/x86/kernel/static_call.c new file +31 -0
@@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> + +static void __static_call_transform (void *insn,u8 opcode,void *func) +{ +const void *code =text_gen_insn (opcode, insn, func); + +if (WARN_ONCE (*(u8 *)insn != opcode, +"unexpected static call insn opcode 0x%x at %pS\n" , + opcode, insn)) +return ; + +if (memcmp (insn, code,CALL_INSN_SIZE ) ==0 ) +return ; + +text_poke_bp (insn, code,CALL_INSN_SIZE ,NULL ); +} + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +mutex_lock (&text_mutex); + +if (tramp) +__static_call_transform (tramp,JMP32_INSN_OPCODE , func); + +mutex_unlock (&text_mutex); +} +EXPORT_SYMBOL_GPL (arch_static_call_transform);
Peter Zijlstra
maintainerPATCH v7 10/18 x86/static_call: Add inline static call implementation for x86-64
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the inline static call implementation for x86-64. The generated code is identical to the out-of-line case, except we move the trampoline into it's own section.
Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.
During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.
--- arch/x86/Kconfig | 3 arch/x86/include/asm/static_call.h | 13 ++- arch/x86/kernel/static_call.c | 3 arch/x86/kernel/vmlinux.lds.S | 1 include/asm-generic/vmlinux.lds.h | 6 + tools/include/linux/static_call_types.h | 28 ++++++ tools/objtool/check.c | 130 ++++++++++++++++++++++++++++++++ tools/objtool/check.h | 1 tools/objtool/elf.c | 8 - tools/objtool/elf.h | 3 tools/objtool/objtool.h | 1 tools/objtool/orc_gen.c | 4 tools/objtool/sync-check.sh | 1 13 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 tools/objtool/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -214,6 +214,7 @@ config X86 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL + select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
@@ -229,6 +230,7 @@ config X86 select RTC_MC146818_LIB select SPARSE_IRQ select SRCU + select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE) select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT
@@ -450,7 +452,6 @@ config GOLDFISH config RETPOLINE bool "Avoid speculative indirect branches in kernel" default y - select STACK_VALIDATION if HAVE_STACK_VALIDATION help Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +12 -1
@@ -5,12 +5,23 @@#include <asm/text-patching.h> /* +* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which +* uses the current value of the key->func pointer to do an indirect jump to +* the function. This trampoline is only used during boot, before the call +* sites get patched by static_call_update(). The name of this trampoline has +* a magical aspect: objtool uses it to find static call sites so it can create +* the .static_call_sites section. +* * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which * does a direct jump to the function. The direct jump gets patched by * static_call_update(). +* +* Having the trampoline in a special section forces GCC to emit a JMP.d32 when +* it does tail-call optimization on the call; since you cannot compute the +* relative displacement across sections. */ #define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ - asm(".pushsection .text, \"ax\" \n" \ + asm(".pushsection .static_call.text, \"ax\" \n" \ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \
arch/x86/kernel/static_call.c +3 -0
@@ -26,6 +26,9 @@ void arch_static_call_transform(void *siif (tramp)__static_call_transform (tramp,JMP32_INSN_OPCODE , func); +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) +__static_call_transform (site,CALL_INSN_OPCODE , func); +mutex_unlock (&text_mutex); }EXPORT_SYMBOL_GPL (arch_static_call_transform);
arch/x86/kernel/vmlinux.lds.S +1 -0
@@ -136,6 +136,7 @@ SECTIONS ENTRY_TEXT ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT + STATIC_CALL_TEXT *(.fixup) *(.gnu.warning)
include/asm-generic/vmlinux.lds.h +6 -0
@@ -620,6 +620,12 @@ *(.softirqentry.text) \ __softirqentry_text_end = .; +#define STATIC_CALL_TEXT \ + ALIGN_FUNCTION(); \ + __static_call_text_start = .; \ + *(.static_call.text) \ + __static_call_text_end = .; +/* Section used for early init (in .S files) */ #define HEAD_TEXT KEEP(*(.head.text))
tools/include/linux/static_call_types.h new file +28 -0
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/types.h> +#include <linux/stringify.h> + +#define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) +#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1) +#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name) + +#define STATIC_CALL_TRAMP_PREFIX __SCT__ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1) +#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; + +#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/check.c +130 -0
172 lines not on this page · raw message
tools/objtool/check.h +1 -0
@@ -22,6 +22,7 @@ struct insn_state {struct instruction {struct list_head list;struct hlist_node hash; +struct list_head static_call_node;struct section *sec;unsigned long offset;unsigned int len;
tools/objtool/elf.c +4 -4
@@ -652,7 +652,7 @@ struct elf *elf_open_read(const char *na }struct section *elf_create_section (struct elf *elf,const char *name, -size_t entsize,int nr) +unsigned int sh_flags,size_t entsize,int nr) {struct section *sec, *shstrtab;size_t size = entsize * nr;
@@ -712,7 +712,7 @@ struct section *elf_create_section(struc sec->sh.sh_entsize = entsize; sec->sh.sh_type =SHT_PROGBITS ; sec->sh.sh_addralign =1 ; - sec->sh.sh_flags =SHF_ALLOC ; + sec->sh.sh_flags =SHF_ALLOC | sh_flags;/* Add section name to .shstrtab (or .strtab for Clang) */
@@ -767,7 +767,7 @@ static struct section *elf_create_rel_restrcpy (relocname,".rel" );strcat (relocname, base->name); - sec =elf_create_section (elf, relocname,sizeof (GElf_Rel),0 ); + sec =elf_create_section (elf, relocname,0 ,sizeof (GElf_Rel),0 );free (relocname);if (!sec)return NULL ;
@@ -797,7 +797,7 @@ static struct section *elf_create_rela_rstrcpy (relocname,".rela" );strcat (relocname, base->name); - sec =elf_create_section (elf, relocname,sizeof (GElf_Rela),0 ); + sec =elf_create_section (elf, relocname,0 ,sizeof (GElf_Rela),0 );free (relocname);if (!sec)return NULL ;
tools/objtool/elf.h +2 -1
@@ -56,6 +56,7 @@ struct symbol {unsigned int len;struct symbol *pfunc, *cfunc, *alias;bool uaccess_safe; +bool static_call_tramp; };struct reloc {
@@ -120,7 +121,7 @@ static inline u32 reloc_hash(struct relo }struct elf *elf_open_read (const char *name,int flags); -struct section *elf_create_section (struct elf *elf,const char *name,size_t entsize,int nr); +struct section *elf_create_section (struct elf *elf,const char *name,unsigned int sh_flags,size_t entsize,int nr);struct section *elf_create_reloc_section (struct elf *elf,struct section *base,int reltype);void elf_add_reloc (struct elf *elf,struct reloc *reloc); int elf_write_insn(struct elf *elf, struct section *sec,
tools/objtool/objtool.h +1 -0
@@ -16,6 +16,7 @@ struct objtool_file {struct elf *elf;struct list_head insn_list;DECLARE_HASHTABLE (insn_hash,20 ); +struct list_head static_call_list;bool ignore_unreachables, c_file, hints, rodata; };
tools/objtool/orc_gen.c +2 -2
@@ -177,7 +177,7 @@ int create_orc_sections(struct objtool_f/* create .orc_unwind_ip and .rela.orc_unwind_ip sections */ - sec =elf_create_section (file->elf,".orc_unwind_ip" ,sizeof (int ), idx); + sec =elf_create_section (file->elf,".orc_unwind_ip" ,0 ,sizeof (int ), idx);if (!sec)return -1 ;
@@ -186,7 +186,7 @@ int create_orc_sections(struct objtool_freturn -1 ;/* create .orc_unwind section */ - u_sec =elf_create_section (file->elf,".orc_unwind" , + u_sec =elf_create_section (file->elf,".orc_unwind" ,0 ,sizeof (struct orc_entry ), idx);/* populate sections */
tools/objtool/sync-check.sh +1 -0
@@ -7,6 +7,7 @@ arch/x86/include/asm/orc_types.h arch/x86/include/asm/emulate_prefix.h arch/x86/lib/x86-opcode-map.txt arch/x86/tools/gen-insn-attr-x86.awk +include/linux/static_call_types.h ' check_2 () {
[tip: core/static_call] x86/static_call: Add inline static call implementation for x86-64
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 1e7e47883830aae5e8246a22ca2fc6883c61acdf Gitweb: https://git.kernel.org/tip/1e7e47883830aae5e8246a22ca2fc6883c61acdf Author: Josh Poimboeuf <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:45 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:05 +02:00
x86/static_call: Add inline static call implementation for x86-64
Add the inline static call implementation for x86-64. The generated code is identical to the out-of-line case, except we move the trampoline into it's own section.
Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.
During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.
[peterz: merged trampolines, put trampoline in section]
--- arch/x86/Kconfig | 3 +- arch/x86/include/asm/static_call.h | 13 +- arch/x86/kernel/static_call.c | 3 +- arch/x86/kernel/vmlinux.lds.S | 1 +- include/asm-generic/vmlinux.lds.h | 6 +- tools/include/linux/static_call_types.h | 28 +++++- tools/objtool/check.c | 130 +++++++++++++++++++++++- tools/objtool/check.h | 1 +- tools/objtool/elf.c | 8 +- tools/objtool/elf.h | 3 +- tools/objtool/objtool.h | 1 +- tools/objtool/orc_gen.c | 4 +- tools/objtool/sync-check.sh | 1 +- 13 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 tools/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -216,6 +216,7 @@ config X86 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL + select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
@@ -231,6 +232,7 @@ config X86 select RTC_MC146818_LIB select SPARSE_IRQ select SRCU + select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE) select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT
@@ -452,7 +454,6 @@ config GOLDFISH config RETPOLINE bool "Avoid speculative indirect branches in kernel" default y - select STACK_VALIDATION if HAVE_STACK_VALIDATION help Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +12 -1
@@ -5,12 +5,23 @@#include <asm/text-patching.h> /* +* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which +* uses the current value of the key->func pointer to do an indirect jump to +* the function. This trampoline is only used during boot, before the call +* sites get patched by static_call_update(). The name of this trampoline has +* a magical aspect: objtool uses it to find static call sites so it can create +* the .static_call_sites section. +* * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which * does a direct jump to the function. The direct jump gets patched by * static_call_update(). +* +* Having the trampoline in a special section forces GCC to emit a JMP.d32 when +* it does tail-call optimization on the call; since you cannot compute the +* relative displacement across sections. */ #define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ - asm(".pushsection .text, \"ax\" \n" \ + asm(".pushsection .static_call.text, \"ax\" \n" \ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \
arch/x86/kernel/static_call.c +3 -0
@@ -26,6 +26,9 @@ void arch_static_call_transform(void *site, void *tramp, void *func)if (tramp)__static_call_transform (tramp,JMP32_INSN_OPCODE , func); +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) +__static_call_transform (site,CALL_INSN_OPCODE , func); +mutex_unlock (&text_mutex); }EXPORT_SYMBOL_GPL (arch_static_call_transform);
arch/x86/kernel/vmlinux.lds.S +1 -0
@@ -136,6 +136,7 @@ SECTIONS ENTRY_TEXT ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT + STATIC_CALL_TEXT *(.fixup) *(.gnu.warning)
include/asm-generic/vmlinux.lds.h +6 -0
@@ -642,6 +642,12 @@ *(.softirqentry.text) \ __softirqentry_text_end = .; +#define STATIC_CALL_TEXT \ + ALIGN_FUNCTION(); \ + __static_call_text_start = .; \ + *(.static_call.text) \ + __static_call_text_end = .; +/* Section used for early init (in .S files) */ #define HEAD_TEXT KEEP(*(.head.text))
tools/include/linux/static_call_types.h new file +28 -0
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/types.h> +#include <linux/stringify.h> + +#define STATIC_CALL_KEY_PREFIX __SCK__ +#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) +#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1) +#define STATIC_CALL_KEY (name) __PASTE(STATIC_CALL_KEY_PREFIX, name) + +#define STATIC_CALL_TRAMP_PREFIX __SCT__ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) +#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1) +#define STATIC_CALL_TRAMP (name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; + +#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/check.c +130 -0
172 lines not on this page · raw message
tools/objtool/check.h +1 -0
@@ -22,6 +22,7 @@ struct insn_state {struct instruction {struct list_head list;struct hlist_node hash; +struct list_head static_call_node;struct section *sec;unsigned long offset;unsigned int len;
tools/objtool/elf.c +4 -4
@@ -652,7 +652,7 @@ err: }struct section *elf_create_section (struct elf *elf,const char *name, -size_t entsize,int nr) +unsigned int sh_flags,size_t entsize,int nr) {struct section *sec, *shstrtab;size_t size = entsize * nr;
@@ -712,7 +712,7 @@ struct section *elf_create_section(struct elf *elf, const char *name, sec->sh.sh_entsize = entsize; sec->sh.sh_type =SHT_PROGBITS ; sec->sh.sh_addralign =1 ; - sec->sh.sh_flags =SHF_ALLOC ; + sec->sh.sh_flags =SHF_ALLOC | sh_flags;/* Add section name to .shstrtab (or .strtab for Clang) */
@@ -767,7 +767,7 @@ static struct section *elf_create_rel_reloc_section(struct elf *elf, struct sectstrcpy (relocname,".rel" );strcat (relocname, base->name); - sec =elf_create_section (elf, relocname,sizeof (GElf_Rel),0 ); + sec =elf_create_section (elf, relocname,0 ,sizeof (GElf_Rel),0 );free (relocname);if (!sec)return NULL ;
@@ -797,7 +797,7 @@ static struct section *elf_create_rela_reloc_section(struct elf *elf, struct secstrcpy (relocname,".rela" );strcat (relocname, base->name); - sec =elf_create_section (elf, relocname,sizeof (GElf_Rela),0 ); + sec =elf_create_section (elf, relocname,0 ,sizeof (GElf_Rela),0 );free (relocname);if (!sec)return NULL ;
tools/objtool/elf.h +2 -1
@@ -56,6 +56,7 @@ struct symbol {unsigned int len;struct symbol *pfunc, *cfunc, *alias;bool uaccess_safe; +bool static_call_tramp; };struct reloc {
@@ -120,7 +121,7 @@ static inline u32 reloc_hash(struct reloc *reloc) }struct elf *elf_open_read (const char *name,int flags); -struct section *elf_create_section (struct elf *elf,const char *name,size_t entsize,int nr); +struct section *elf_create_section (struct elf *elf,const char *name,unsigned int sh_flags,size_t entsize,int nr);struct section *elf_create_reloc_section (struct elf *elf,struct section *base,int reltype);void elf_add_reloc (struct elf *elf,struct reloc *reloc); int elf_write_insn(struct elf *elf, struct section *sec,
tools/objtool/objtool.h +1 -0
@@ -16,6 +16,7 @@ struct objtool_file {struct elf *elf;struct list_head insn_list;DECLARE_HASHTABLE (insn_hash,20 ); +struct list_head static_call_list;bool ignore_unreachables, c_file, hints, rodata; };
tools/objtool/orc_gen.c +2 -2
@@ -177,7 +177,7 @@ int create_orc_sections(struct objtool_file *file)/* create .orc_unwind_ip and .rela.orc_unwind_ip sections */ - sec =elf_create_section (file->elf,".orc_unwind_ip" ,sizeof (int ), idx); + sec =elf_create_section (file->elf,".orc_unwind_ip" ,0 ,sizeof (int ), idx);if (!sec)return -1 ;
@@ -186,7 +186,7 @@ int create_orc_sections(struct objtool_file *file)return -1 ;/* create .orc_unwind section */ - u_sec =elf_create_section (file->elf,".orc_unwind" , + u_sec =elf_create_section (file->elf,".orc_unwind" ,0 ,sizeof (struct orc_entry ), idx);/* populate sections */
tools/objtool/sync-check.sh +1 -0
@@ -7,6 +7,7 @@ arch/x86/include/asm/orc_types.h arch/x86/include/asm/emulate_prefix.h arch/x86/lib/x86-opcode-map.txt arch/x86/tools/gen-insn-attr-x86.awk +include/linux/static_call_types.h ' check_2 () {
Peter Zijlstra
maintainerPATCH v7 11/18 static_call: Simple self-test
In reply to Peter Zijlstra--- arch/Kconfig | 6 ++++++ kernel/static_call.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+)
arch/Kconfig +6 -0
@@ -103,6 +103,12 @@ config STATIC_KEYS_SELFTEST help Boot time self-test of the branch patching code. +config STATIC_CALL_SELFTEST + bool "Static call selftest" + depends on HAVE_STATIC_CALL + help + Boot time self-test of the call patching code. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES
kernel/static_call.c +43 -0
46 lines not on this page · raw message
[tip: core/static_call] static_call: Add simple self-test for static calls
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: f03c412915f5f69f2d17bcd20ecdd69320bcbf7b Gitweb: https://git.kernel.org/tip/f03c412915f5f69f2d17bcd20ecdd69320bcbf7b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:46 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:05 +02:00
static_call: Add simple self-test for static calls
--- arch/Kconfig | 6 ++++++- kernel/static_call.c | 43 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+)
arch/Kconfig +6 -0
@@ -106,6 +106,12 @@ config STATIC_KEYS_SELFTEST help Boot time self-test of the branch patching code. +config STATIC_CALL_SELFTEST + bool "Static call selftest" + depends on HAVE_STATIC_CALL + help + Boot time self-test of the call patching code. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES
kernel/static_call.c +43 -0
46 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v7 12/18 x86/alternatives: Teach text_poke_bp() to emulate RET
In reply to Peter ZijlstraFuture patches will need to poke a RET instruction, provide the infrastructure required for this.
--- arch/x86/include/asm/text-patching.h | 19 +++++++++++++++++++ arch/x86/kernel/alternative.c | 5 +++++ 2 files changed, 24 insertions(+)
arch/x86/include/asm/text-patching.h +19 -0
@@ -53,6 +53,9 @@ extern void text_poke_finish(void);#define INT3_INSN_SIZE 1#define INT3_INSN_OPCODE 0xCC +#define RET_INSN_SIZE 1 +#define RET_INSN_OPCODE 0xC3 +#define CALL_INSN_SIZE 5#define CALL_INSN_OPCODE 0xE8
@@ -73,6 +76,7 @@ static __always_inline int text_opcode_sswitch (opcode) {__CASE (INT3 ); +__CASE (RET );__CASE (CALL );__CASE (JMP32 );__CASE (JMP8 );
@@ -141,11 +145,26 @@ void int3_emulate_push(struct pt_regs *r }static __always_inline +unsigned long int3_emulate_pop (struct pt_regs *regs) +{ +unsigned long val = *(unsigned long *)regs->sp; + regs->sp +=sizeof (unsigned long ); +return val; +} + +static __always_inline void int3_emulate_call(struct pt_regs *regs, unsigned long func) {int3_emulate_push (regs, regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE );int3_emulate_jmp (regs, func); } + +static __always_inline +void int3_emulate_ret(struct pt_regs *regs) +{ +unsigned long ip =int3_emulate_pop (regs); +int3_emulate_jmp (regs, ip); +}#endif /* !CONFIG_UML_X86 */ #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/alternative.c +5 -0
@@ -1099,6 +1099,10 @@ int noinstr poke_int3_handler(struct pt_*/ goto out_put; +case RET_INSN_OPCODE : +int3_emulate_ret (regs); +break ; +case CALL_INSN_OPCODE :int3_emulate_call (regs, (long )ip + tp->rel32);break ;
@@ -1240,6 +1244,7 @@ static void text_poke_loc_init(struct teswitch (tp->opcode) {case INT3_INSN_OPCODE : +case RET_INSN_OPCODE :break ;case CALL_INSN_OPCODE :
[tip: core/static_call] x86/alternatives: Teach text_poke_bp() to emulate RET
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: c43a43e439e00ad2a4d98716895d961ade6bbbfc Gitweb: https://git.kernel.org/tip/c43a43e439e00ad2a4d98716895d961ade6bbbfc Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:47 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:05 +02:00
x86/alternatives: Teach text_poke_bp() to emulate RET
Future patches will need to poke a RET instruction, provide the infrastructure required for this.
--- arch/x86/include/asm/text-patching.h | 19 +++++++++++++++++++ arch/x86/kernel/alternative.c | 5 +++++ 2 files changed, 24 insertions(+)
arch/x86/include/asm/text-patching.h +19 -0
@@ -53,6 +53,9 @@ extern void text_poke_finish(void);#define INT3_INSN_SIZE 1#define INT3_INSN_OPCODE 0xCC +#define RET_INSN_SIZE 1 +#define RET_INSN_OPCODE 0xC3 +#define CALL_INSN_SIZE 5#define CALL_INSN_OPCODE 0xE8
@@ -73,6 +76,7 @@ static __always_inline int text_opcode_size(u8 opcode)switch (opcode) {__CASE (INT3 ); +__CASE (RET );__CASE (CALL );__CASE (JMP32 );__CASE (JMP8 );
@@ -141,11 +145,26 @@ void int3_emulate_push(struct pt_regs *regs, unsigned long val) }static __always_inline +unsigned long int3_emulate_pop (struct pt_regs *regs) +{ +unsigned long val = *(unsigned long *)regs->sp; + regs->sp +=sizeof (unsigned long ); +return val; +} + +static __always_inline void int3_emulate_call(struct pt_regs *regs, unsigned long func) {int3_emulate_push (regs, regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE );int3_emulate_jmp (regs, func); } + +static __always_inline +void int3_emulate_ret(struct pt_regs *regs) +{ +unsigned long ip =int3_emulate_pop (regs); +int3_emulate_jmp (regs, ip); +}#endif /* !CONFIG_UML_X86 */ #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/alternative.c +5 -0
@@ -1103,6 +1103,10 @@ noinstr int poke_int3_handler(struct pt_regs *regs)*/ goto out_put; +case RET_INSN_OPCODE : +int3_emulate_ret (regs); +break ; +case CALL_INSN_OPCODE :int3_emulate_call (regs, (long )ip + tp->rel32);break ;
@@ -1277,6 +1281,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,switch (tp->opcode) {case INT3_INSN_OPCODE : +case RET_INSN_OPCODE :break ;case CALL_INSN_OPCODE :
Peter Zijlstra
maintainerPATCH v7 13/18 static_call: Add static_call_cond()
In reply to Peter ZijlstraExtend the static_call infrastructure to optimize the following common pattern:
if (func_ptr) func_ptr(args...)
For the trampoline (which is in effect a tail-call), we patch the JMP.d32 into a RET, which then directly consumes the trampoline call.
For the in-line sites we replace the CALL with a NOP5.
NOTE: this is 'obviously' limited to functions with a 'void' return type.
NOTE: DEFINE_STATIC_COND_CALL() only requires a typename, as opposed
to a full function.
--- arch/x86/include/asm/static_call.h | 12 +++-- arch/x86/kernel/static_call.c | 42 +++++++++++++----- include/linux/static_call.h | 86 +++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 13 deletions(-)
arch/x86/include/asm/static_call.h +9 -3
@@ -20,15 +20,21 @@* it does tail-call optimization on the call; since you cannot compute the * relative displacement across sections. */ -#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + +#define __ARCH_DEFINE_STATIC_CALL_TRAMP (name, insns) \ asm(".pushsection .static_call.text, \"ax\" \n" \ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \ - " .byte 0xe9 # jmp.d32 \n" \ - " .long " #func " - (. + 4) \n" \ + insns " \n" \ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ ".popsection \n") +#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)") + +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP (name) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/static_call.c +32 -10
61 lines not on this page · raw message
include/linux/static_call.h +86 -0
122 lines not on this page · raw message
[tip: core/static_call] static_call: Add static_call_cond()
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 452cddbff74b6a15b9354505671011700fe03710 Gitweb: https://git.kernel.org/tip/452cddbff74b6a15b9354505671011700fe03710 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:48 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:05 +02:00
static_call: Add static_call_cond()
Extend the static_call infrastructure to optimize the following common pattern:
if (func_ptr) func_ptr(args...)
For the trampoline (which is in effect a tail-call), we patch the JMP.d32 into a RET, which then directly consumes the trampoline call.
For the in-line sites we replace the CALL with a NOP5.
NOTE: this is 'obviously' limited to functions with a 'void' return type.
NOTE: DEFINE_STATIC_COND_CALL() only requires a typename, as opposed
to a full function.
--- arch/x86/include/asm/static_call.h | 12 +++- arch/x86/kernel/static_call.c | 42 ++++++++++---- include/linux/static_call.h | 86 +++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 13 deletions(-)
arch/x86/include/asm/static_call.h +9 -3
@@ -20,15 +20,21 @@* it does tail-call optimization on the call; since you cannot compute the * relative displacement across sections. */ -#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + +#define __ARCH_DEFINE_STATIC_CALL_TRAMP (name, insns) \ asm(".pushsection .static_call.text, \"ax\" \n" \ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \ - " .byte 0xe9 # jmp.d32 \n" \ - " .long " #func " - (. + 4) \n" \ + insns " \n" \ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ ".popsection \n") +#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)") + +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP (name) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/static_call.c +32 -10
61 lines not on this page · raw message
include/linux/static_call.h +86 -0
122 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v7 14/18 static_call: Handle tail-calls
In reply to Peter ZijlstraGCC can turn our static_call(name)(args...) into a tail call, in which case we get a JMP.d32 into the trampoline (which then does a further tail-call).
Teach objtool to recognise and mark these in .static_call_sites and adjust the code patching to deal with this.
--- arch/x86/kernel/static_call.c | 21 ++++++++++++++++++--- include/linux/static_call.h | 4 ++-- include/linux/static_call_types.h | 7 +++++++ kernel/static_call.c | 21 +++++++++++++-------- tools/include/linux/static_call_types.h | 7 +++++++ tools/objtool/check.c | 18 +++++++++++++----- 6 files changed, 60 insertions(+), 18 deletions(-)
arch/x86/kernel/static_call.c +18 -3
@@ -41,15 +41,30 @@ static void __static_call_transform(voidtext_poke_bp (insn, code, size,NULL ); } -void arch_static_call_transform (void *site,void *tramp,void *func) +static inline enum insn_type __sc_insn (bool null,bool tail) +{ +/* +* Encode the following table without branches: +* +* tail null insn +* -----+-------+------ +* 0 | 0 | CALL +* 0 | 1 | NOP +* 1 | 0 | JMP +* 1 | 1 | RET +*/ +return 2 *tail + null; +} + +void arch_static_call_transform (void *site,void *tramp,void *func,bool tail) {mutex_lock (&text_mutex);if (tramp) -__static_call_transform (tramp, func ?JMP :RET , func); +__static_call_transform (tramp,__sc_insn (!func, true), func);if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) -__static_call_transform (site, func ?CALL :NOP , func); +__static_call_transform (site,__sc_insn (!func, tail), func);mutex_unlock (&text_mutex); }
include/linux/static_call.h +2 -2
@@ -103,7 +103,7 @@/* * Either @site or @tramp can be NULL. */ -extern void arch_static_call_transform (void *site,void *tramp,void *func); +extern void arch_static_call_transform (void *site,void *tramp,void *func,bool tail);#define STATIC_CALL_TRAMP_ADDR (name) &STATIC_CALL_TRAMP(name)
@@ -206,7 +206,7 @@ void __static_call_update(struct static_ {cpus_read_lock ();WRITE_ONCE (key->func, func); -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);cpus_read_unlock (); }
include/linux/static_call_types.h +7 -0
@@ -17,6 +17,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
kernel/static_call.c +13 -8
@@ -15,8 +15,6 @@ extern struct static_call_site __start_sstatic bool static_call_initialized; -#define STATIC_CALL_INIT 1UL -/* mutex to protect key modules/sites */ static DEFINE_MUTEX (static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(strstatic inline struct static_call_key *static_call_key (const struct static_call_site *site) {return (struct static_call_key *) - (((long )site->key + (long )&site->key) & ~STATIC_CALL_INIT ); + (((long )site->key + (long )&site->key) & ~STATIC_CALL_SITE_FLAGS ); }/* These assume the key is word-aligned. */ static inline bool static_call_is_init (struct static_call_site *site) { -return ((long )site->key + (long )&site->key) &STATIC_CALL_INIT ; +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_INIT ; +} + +static inline bool static_call_is_tail (struct static_call_site *site) +{ +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_TAIL ; }static inline void static_call_set_init (struct static_call_site *site) { - site->key = ((long )static_call_key (site) |STATIC_CALL_INIT ) - + site->key = ((long )static_call_key (site) |STATIC_CALL_SITE_INIT ) - (long )&site->key; }
@@ -104,7 +107,7 @@ void __static_call_update(struct static_ key->func = func; -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);/* * If uninitialized, we'll not update the callsites, but they still
@@ -154,7 +157,8 @@ void __static_call_update(struct static_continue ; } -arch_static_call_transform (site_addr,NULL , func); +arch_static_call_transform (site_addr,NULL , func, +static_call_is_tail (site)); } }
@@ -198,7 +202,8 @@ static int __static_call_init(struct mod key->mods = site_mod; } -arch_static_call_transform (site_addr,NULL , key->func); +arch_static_call_transform (site_addr,NULL , key->func, +static_call_is_tail (site)); }return 0 ;
tools/include/linux/static_call_types.h +7 -0
@@ -17,6 +17,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
tools/objtool/check.c +13 -5
@@ -511,7 +511,7 @@ static int create_static_call_sections(s }memset (reloc,0 ,sizeof (*reloc)); reloc->sym = key_sym; - reloc->addend =0 ; + reloc->addend =is_sibling_call (insn) ?STATIC_CALL_SITE_TAIL :0 ; reloc->type =R_X86_64_PC32 ; reloc->offset = idx *sizeof (struct static_call_site ) +4 ; reloc->sec = reloc_sec;
@@ -720,6 +720,10 @@ static int add_jump_destinations(struct }else {/* external sibling call */ insn->call_dest = reloc->sym; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + }continue ; }
@@ -771,6 +775,10 @@ static int add_jump_destinations(struct/* internal sibling call */ insn->call_dest = insn->jump_dest->func; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + } } } }
@@ -1639,6 +1647,10 @@ static int decode_sections(struct objtooif (ret)return ret; + ret =read_static_call_tramps (file); +if (ret) +return ret; + ret =add_jump_destinations (file);if (ret)return ret;
@@ -1671,10 +1683,6 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =read_static_call_tramps (file); -if (ret) -return ret; -return 0 ; }
[tip: core/static_call] static_call: Handle tail-calls
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997 Gitweb: https://git.kernel.org/tip/5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:49 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
static_call: Handle tail-calls
GCC can turn our static_call(name)(args...) into a tail call, in which case we get a JMP.d32 into the trampoline (which then does a further tail-call).
Teach objtool to recognise and mark these in .static_call_sites and adjust the code patching to deal with this.
--- arch/x86/kernel/static_call.c | 21 ++++++++++++++++++--- include/linux/static_call.h | 4 ++-- include/linux/static_call_types.h | 7 +++++++ kernel/static_call.c | 21 +++++++++++++-------- tools/include/linux/static_call_types.h | 7 +++++++ tools/objtool/check.c | 18 +++++++++++++----- 6 files changed, 60 insertions(+), 18 deletions(-)
arch/x86/kernel/static_call.c +18 -3
@@ -41,15 +41,30 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func)text_poke_bp (insn, code, size,NULL ); } -void arch_static_call_transform (void *site,void *tramp,void *func) +static inline enum insn_type __sc_insn (bool null,bool tail) +{ +/* +* Encode the following table without branches: +* +* tail null insn +* -----+-------+------ +* 0 | 0 | CALL +* 0 | 1 | NOP +* 1 | 0 | JMP +* 1 | 1 | RET +*/ +return 2 *tail + null; +} + +void arch_static_call_transform (void *site,void *tramp,void *func,bool tail) {mutex_lock (&text_mutex);if (tramp) -__static_call_transform (tramp, func ?JMP :RET , func); +__static_call_transform (tramp,__sc_insn (!func, true), func);if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) -__static_call_transform (site, func ?CALL :NOP , func); +__static_call_transform (site,__sc_insn (!func, tail), func);mutex_unlock (&text_mutex); }
include/linux/static_call.h +2 -2
@@ -103,7 +103,7 @@/* * Either @site or @tramp can be NULL. */ -extern void arch_static_call_transform (void *site,void *tramp,void *func); +extern void arch_static_call_transform (void *site,void *tramp,void *func,bool tail);#define STATIC_CALL_TRAMP_ADDR (name) &STATIC_CALL_TRAMP(name)
@@ -206,7 +206,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) {cpus_read_lock ();WRITE_ONCE (key->func, func); -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);cpus_read_unlock (); }
include/linux/static_call_types.h +7 -0
@@ -17,6 +17,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
kernel/static_call.c +13 -8
@@ -15,8 +15,6 @@ extern struct static_call_site __start_static_call_sites[],static bool static_call_initialized; -#define STATIC_CALL_INIT 1UL -/* mutex to protect key modules/sites */ static DEFINE_MUTEX (static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(struct static_call_site *site)static inline struct static_call_key *static_call_key (const struct static_call_site *site) {return (struct static_call_key *) - (((long )site->key + (long )&site->key) & ~STATIC_CALL_INIT ); + (((long )site->key + (long )&site->key) & ~STATIC_CALL_SITE_FLAGS ); }/* These assume the key is word-aligned. */ static inline bool static_call_is_init (struct static_call_site *site) { -return ((long )site->key + (long )&site->key) &STATIC_CALL_INIT ; +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_INIT ; +} + +static inline bool static_call_is_tail (struct static_call_site *site) +{ +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_TAIL ; }static inline void static_call_set_init (struct static_call_site *site) { - site->key = ((long )static_call_key (site) |STATIC_CALL_INIT ) - + site->key = ((long )static_call_key (site) |STATIC_CALL_SITE_INIT ) - (long )&site->key; }
@@ -104,7 +107,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func) key->func = func; -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);/* * If uninitialized, we'll not update the callsites, but they still
@@ -154,7 +157,8 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)continue ; } -arch_static_call_transform (site_addr,NULL , func); +arch_static_call_transform (site_addr,NULL , func, +static_call_is_tail (site)); } }
@@ -198,7 +202,8 @@ static int __static_call_init(struct module *mod, key->mods = site_mod; } -arch_static_call_transform (site_addr,NULL , key->func); +arch_static_call_transform (site_addr,NULL , key->func, +static_call_is_tail (site)); }return 0 ;
tools/include/linux/static_call_types.h +7 -0
@@ -17,6 +17,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
tools/objtool/check.c +13 -5
@@ -516,7 +516,7 @@ static int create_static_call_sections(struct objtool_file *file) }memset (reloc,0 ,sizeof (*reloc)); reloc->sym = key_sym; - reloc->addend =0 ; + reloc->addend =is_sibling_call (insn) ?STATIC_CALL_SITE_TAIL :0 ; reloc->type =R_X86_64_PC32 ; reloc->offset = idx *sizeof (struct static_call_site ) +4 ; reloc->sec = reloc_sec;
@@ -747,6 +747,10 @@ static int add_jump_destinations(struct objtool_file *file) }else {/* external sibling call */ insn->call_dest = reloc->sym; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + }continue ; }
@@ -798,6 +802,10 @@ static int add_jump_destinations(struct objtool_file *file)/* internal sibling call */ insn->call_dest = insn->jump_dest->func; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + } } } }
@@ -1684,6 +1692,10 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; + ret =read_static_call_tramps (file); +if (ret) +return ret; + ret =add_jump_destinations (file);if (ret)return ret;
@@ -1716,10 +1728,6 @@ static int decode_sections(struct objtool_file *file)if (ret)return ret; - ret =read_static_call_tramps (file); -if (ret) -return ret; -return 0 ; }
Peter Zijlstra
maintainerPATCH v7 15/18 static_call: Add some validation
In reply to Peter ZijlstraVerify the text we're about to change is as we expect it to be.
--- arch/x86/kernel/static_call.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
arch/x86/kernel/static_call.c +26 -2
@@ -41,6 +41,26 @@ static void __static_call_transform(voidtext_poke_bp (insn, code, size,NULL ); } +static void __static_call_validate (void *insn,bool tail) +{ +u8 opcode = *(u8 *)insn; + +if (tail) { +if (opcode ==JMP32_INSN_OPCODE || + opcode ==RET_INSN_OPCODE ) +return ; + }else { +if (opcode ==CALL_INSN_OPCODE || + !memcmp (insn, ideal_nops[NOP_ATOMIC5 ],5 )) +return ; + } + +/* +* If we ever trigger this, our text is corrupt, we'll probably not live long. +*/ +WARN_ONCE (1 ,"unexpected static_call insn opcode 0x%x at %pS\n" , opcode, insn); +} +static inline enum insn_type __sc_insn (bool null,bool tail) {/*
@@ -60,11 +80,15 @@ void arch_static_call_transform(void *si {mutex_lock (&text_mutex); -if (tramp) +if (tramp) { +__static_call_validate (tramp, true);__static_call_transform (tramp,__sc_insn (!func, true), func); + } -if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) { +__static_call_validate (site, tail);__static_call_transform (site,__sc_insn (!func, tail), func); + }mutex_unlock (&text_mutex); }
[tip: core/static_call] static_call: Add some validation
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 6c3fce794e9d2a5ce3a948962d0808a459c40a84 Gitweb: https://git.kernel.org/tip/6c3fce794e9d2a5ce3a948962d0808a459c40a84 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:50 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
static_call: Add some validation
Verify the text we're about to change is as we expect it to be.
--- arch/x86/kernel/static_call.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
arch/x86/kernel/static_call.c +26 -2
@@ -41,6 +41,26 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func)text_poke_bp (insn, code, size,NULL ); } +static void __static_call_validate (void *insn,bool tail) +{ +u8 opcode = *(u8 *)insn; + +if (tail) { +if (opcode ==JMP32_INSN_OPCODE || + opcode ==RET_INSN_OPCODE ) +return ; + }else { +if (opcode ==CALL_INSN_OPCODE || + !memcmp (insn, ideal_nops[NOP_ATOMIC5 ],5 )) +return ; + } + +/* +* If we ever trigger this, our text is corrupt, we'll probably not live long. +*/ +WARN_ONCE (1 ,"unexpected static_call insn opcode 0x%x at %pS\n" , opcode, insn); +} +static inline enum insn_type __sc_insn (bool null,bool tail) {/*
@@ -60,11 +80,15 @@ void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) {mutex_lock (&text_mutex); -if (tramp) +if (tramp) { +__static_call_validate (tramp, true);__static_call_transform (tramp,__sc_insn (!func, true), func); + } -if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) { +__static_call_validate (site, tail);__static_call_transform (site,__sc_insn (!func, tail), func); + }mutex_unlock (&text_mutex); }
Peter Zijlstra
maintainerPATCH v7 16/18 static_call: Allow early init
In reply to Peter ZijlstraIn order to use static_call() to wire up x86_pmu, we need to initialize earlier, specifically before memory allocation works; copy some of the tricks from jump_label to enable this.
Primarily we overload key->next to store a sites pointer when there are no modules, this avoids having to use kmalloc() to initialize the sites and allows us to run much earlier.
--- arch/x86/kernel/setup.c | 2 + arch/x86/kernel/static_call.c | 5 ++- include/linux/static_call.h | 15 +++++++-- kernel/static_call.c | 70 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 7 deletions(-)
arch/x86/kernel/setup.c +2 -0
@@ -19,6 +19,7 @@#include <linux/hugetlb.h> #include <linux/tboot.h> #include <linux/usb/xhci-dbgp.h> +#include <linux/static_call.h> #include <uapi/linux/mount.h>
@@ -849,6 +850,7 @@ void __init setup_arch(char **cmdline_p)early_cpu_init ();arch_init_ideal_nops ();jump_label_init (); +static_call_init ();early_ioremap_init ();setup_olpc_ofw_pgd ();
arch/x86/kernel/static_call.c +4 -1
@@ -11,7 +11,7 @@ enum insn_type { RET = 3, /* tramp / site cond-tail-call */ }; -static void __static_call_transform (void *insn,enum insn_type type,void *func) +static void __ref __static_call_transform (void *insn,enum insn_type type,void *func) {int size =CALL_INSN_SIZE ;const void *code;
@@ -38,6 +38,9 @@ static void __static_call_transform(voidif (memcmp (insn, code, size) ==0 )return ; +if (unlikely (system_state ==SYSTEM_BOOTING )) +return text_poke_early (insn, code, size); +text_poke_bp (insn, code, size,NULL ); }
include/linux/static_call.h +13 -2
@@ -136,6 +136,8 @@ extern void arch_static_call_transform(v #ifdef CONFIG_HAVE_STATIC_CALL_INLINE +extern void __init static_call_init(void); + struct static_call_mod { struct static_call_mod *next; struct module *mod; /* for vmlinux, mod == NULL */
@@ -144,7 +146,12 @@ struct static_call_mod {struct static_call_key {void *func; -struct static_call_mod *mods; +union { +/* bit 0: 0 = mods, 1 = sites */ +unsigned long type; +struct static_call_mod *mods; +struct static_call_site *sites; + }; };extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);
@@ -155,7 +162,7 @@ extern int static_call_text_reserved(voiDECLARE_STATIC_CALL (name, _func); \struct static_call_key STATIC_CALL_KEY (name ) = { \ .func = _func, \ - .mods =NULL , \ + .type =1 , \ }; \ARCH_DEFINE_STATIC_CALL_TRAMP (name, _func)
@@ -180,6 +187,8 @@ extern int static_call_text_reserved(voi#elif defined(CONFIG_HAVE_STATIC_CALL ) +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
@@ -225,6 +234,8 @@ static inline int static_call_text_reser#else /* Generic implementation */ +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
kernel/static_call.c +66 -4
112 lines not on this page · raw message
[tip: core/static_call] static_call: Allow early init
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: a945c8345ec0decb2f1a7f19a8c5e60bcb1dd1eb Gitweb: https://git.kernel.org/tip/a945c8345ec0decb2f1a7f19a8c5e60bcb1dd1eb Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:51 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
static_call: Allow early init
In order to use static_call() to wire up x86_pmu, we need to initialize earlier, specifically before memory allocation works; copy some of the tricks from jump_label to enable this.
Primarily we overload key->next to store a sites pointer when there are no modules, this avoids having to use kmalloc() to initialize the sites and allows us to run much earlier.
--- arch/x86/kernel/setup.c | 2 +- arch/x86/kernel/static_call.c | 5 +- include/linux/static_call.h | 15 ++++++- kernel/static_call.c | 70 ++++++++++++++++++++++++++++++++-- 4 files changed, 85 insertions(+), 7 deletions(-)
arch/x86/kernel/setup.c +2 -0
@@ -19,6 +19,7 @@#include <linux/hugetlb.h> #include <linux/tboot.h> #include <linux/usb/xhci-dbgp.h> +#include <linux/static_call.h> #include <uapi/linux/mount.h>
@@ -849,6 +850,7 @@ void __init setup_arch(char **cmdline_p)early_cpu_init ();arch_init_ideal_nops ();jump_label_init (); +static_call_init ();early_ioremap_init ();setup_olpc_ofw_pgd ();
arch/x86/kernel/static_call.c +4 -1
@@ -11,7 +11,7 @@ enum insn_type { RET = 3, /* tramp / site cond-tail-call */ }; -static void __static_call_transform (void *insn,enum insn_type type,void *func) +static void __ref __static_call_transform (void *insn,enum insn_type type,void *func) {int size =CALL_INSN_SIZE ;const void *code;
@@ -38,6 +38,9 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func)if (memcmp (insn, code, size) ==0 )return ; +if (unlikely (system_state ==SYSTEM_BOOTING )) +return text_poke_early (insn, code, size); +text_poke_bp (insn, code, size,NULL ); }
include/linux/static_call.h +13 -2
@@ -136,6 +136,8 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool #ifdef CONFIG_HAVE_STATIC_CALL_INLINE +extern void __init static_call_init(void); + struct static_call_mod { struct static_call_mod *next; struct module *mod; /* for vmlinux, mod == NULL */
@@ -144,7 +146,12 @@ struct static_call_mod {struct static_call_key {void *func; -struct static_call_mod *mods; +union { +/* bit 0: 0 = mods, 1 = sites */ +unsigned long type; +struct static_call_mod *mods; +struct static_call_site *sites; + }; };extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);
@@ -155,7 +162,7 @@ extern int static_call_text_reserved(void *start, void *end);DECLARE_STATIC_CALL (name, _func); \struct static_call_key STATIC_CALL_KEY (name ) = { \ .func = _func, \ - .mods =NULL , \ + .type =1 , \ }; \ARCH_DEFINE_STATIC_CALL_TRAMP (name, _func)
@@ -180,6 +187,8 @@ extern int static_call_text_reserved(void *start, void *end);#elif defined(CONFIG_HAVE_STATIC_CALL ) +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
@@ -225,6 +234,8 @@ static inline int static_call_text_reserved(void *start, void *end)#else /* Generic implementation */ +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
kernel/static_call.c +66 -4
112 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v7 17/18 tracepoint: Optimize using static_call()
In reply to Peter ZijlstraFrom: "Steven Rostedt (VMware)" <[email protected]>
Currently the tracepoint site will iterate a vector and issue indirect calls to however many handlers are registered (ie. the vector is long).
Using static_call() it is possible to optimize this for the common case of only having a single handler registered. In this case the static_call() can directly call this handler. Otherwise, if the vector is longer than 1, call a function that iterates the whole vector like the current code.
--- include/linux/tracepoint-defs.h | 5 ++ include/linux/tracepoint.h | 86 ++++++++++++++++++++++++++++------------ include/trace/define_trace.h | 14 +++--- kernel/tracepoint.c | 25 +++++++++-- 4 files changed, 94 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +5 -0
@@ -11,6 +11,8 @@#include <linux/atomic.h> #include <linux/static_key.h> +struct static_call_key ; +struct trace_print_flags {unsigned long mask;const char *name;
@@ -30,6 +32,9 @@ struct tracepoint_func {struct tracepoint {const char *name;/* Tracepoint name */ struct static_key key; +struct static_call_key *static_call_key; +void *static_call_tramp; +void *iterator;int (*regfunc)(void );void (*unregfunc)(void );struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +61 -25
153 lines not on this page · raw message
include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@#undef TRACE_EVENT#define TRACE_EVENT (name, proto, args, tstruct, assign, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_CONDITION#define TRACE_EVENT_CONDITION (name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@#undef TRACE_EVENT_FN#define TRACE_EVENT_FN (name, proto, args, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_FN_COND#define TRACE_EVENT_FN_COND (name, proto, args, cond, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_NOP#define TRACE_EVENT_NOP (name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@#undef DEFINE_EVENT#define DEFINE_EVENT (template, name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_FN#define DEFINE_EVENT_FN (template, name, proto, args, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_PRINT#define DEFINE_EVENT_PRINT (template, name, proto, args, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_CONDITION#define DEFINE_EVENT_CONDITION (template, name, proto, args, cond) \
@@ -70,7 +70,7 @@#undef DECLARE_TRACE#define DECLARE_TRACE (name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_INCLUDE#undef __TRACE_INCLUDE
kernel/tracepoint.c +21 -4
@@ -221,6 +221,20 @@ static void *func_remove(struct tracepoireturn old; } +static void tracepoint_update_call (struct tracepoint *tp,struct tracepoint_func *tp_funcs) +{ +void *func = tp->iterator; + +/* Synthetic events do not have static call sites */ +if (!tp->static_call_key) +return ; + +if (!tp_funcs[1 ].func) + func = tp_funcs[0 ].func; + +__static_call_update (tp->static_call_key, tp->static_call_tramp, func); +} +/* * Add the probe function to a tracepoint. */
@@ -251,8 +265,9 @@ static int tracepoint_add_func(struct tr* include/linux/tracepoint.h using rcu_dereference_sched(). */ rcu_assign_pointer (tp->funcs, tp_funcs); -if (!static_key_enabled (&tp->key)) -static_key_slow_inc (&tp->key); +tracepoint_update_call (tp, tp_funcs); +static_key_enable (&tp->key); +release_probes (old);return 0 ; }
@@ -281,9 +296,11 @@ static int tracepoint_remove_func(structif (tp->unregfunc &&static_key_enabled (&tp->key)) tp->unregfunc (); -if (static_key_enabled (&tp->key)) -static_key_slow_dec (&tp->key); +static_key_disable (&tp->key); + }else { +tracepoint_update_call (tp, tp_funcs); } +rcu_assign_pointer (tp->funcs, tp_funcs);release_probes (old);return 0 ;
[tip: core/static_call] tracepoint: Optimize using static_call()
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: d25e37d89dd2f41d7acae0429039d2f0ae8b4a07 Gitweb: https://git.kernel.org/tip/d25e37d89dd2f41d7acae0429039d2f0ae8b4a07 Author: Steven Rostedt (VMware) <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:52 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
tracepoint: Optimize using static_call()
Currently the tracepoint site will iterate a vector and issue indirect calls to however many handlers are registered (ie. the vector is long).
Using static_call() it is possible to optimize this for the common case of only having a single handler registered. In this case the static_call() can directly call this handler. Otherwise, if the vector is longer than 1, call a function that iterates the whole vector like the current code.
[peterz: updated to new interface]
--- include/linux/tracepoint-defs.h | 5 ++- include/linux/tracepoint.h | 86 ++++++++++++++++++++++---------- include/trace/define_trace.h | 14 ++--- kernel/tracepoint.c | 25 +++++++-- 4 files changed, 94 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +5 -0
@@ -11,6 +11,8 @@#include <linux/atomic.h> #include <linux/static_key.h> +struct static_call_key ; +struct trace_print_flags {unsigned long mask;const char *name;
@@ -30,6 +32,9 @@ struct tracepoint_func {struct tracepoint {const char *name;/* Tracepoint name */ struct static_key key; +struct static_call_key *static_call_key; +void *static_call_tramp; +void *iterator;int (*regfunc)(void );void (*unregfunc)(void );struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +61 -25
153 lines not on this page · raw message
include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@#undef TRACE_EVENT#define TRACE_EVENT (name, proto, args, tstruct, assign, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_CONDITION#define TRACE_EVENT_CONDITION (name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@#undef TRACE_EVENT_FN#define TRACE_EVENT_FN (name, proto, args, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_FN_COND#define TRACE_EVENT_FN_COND (name, proto, args, cond, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_NOP#define TRACE_EVENT_NOP (name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@#undef DEFINE_EVENT#define DEFINE_EVENT (template, name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_FN#define DEFINE_EVENT_FN (template, name, proto, args, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_PRINT#define DEFINE_EVENT_PRINT (template, name, proto, args, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_CONDITION#define DEFINE_EVENT_CONDITION (template, name, proto, args, cond) \
@@ -70,7 +70,7 @@#undef DECLARE_TRACE#define DECLARE_TRACE (name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_INCLUDE#undef __TRACE_INCLUDE
kernel/tracepoint.c +21 -4
@@ -221,6 +221,20 @@ static void *func_remove(struct tracepoint_func **funcs,return old; } +static void tracepoint_update_call (struct tracepoint *tp,struct tracepoint_func *tp_funcs) +{ +void *func = tp->iterator; + +/* Synthetic events do not have static call sites */ +if (!tp->static_call_key) +return ; + +if (!tp_funcs[1 ].func) + func = tp_funcs[0 ].func; + +__static_call_update (tp->static_call_key, tp->static_call_tramp, func); +} +/* * Add the probe function to a tracepoint. */
@@ -251,8 +265,9 @@ static int tracepoint_add_func(struct tracepoint *tp,* include/linux/tracepoint.h using rcu_dereference_sched(). */ rcu_assign_pointer (tp->funcs, tp_funcs); -if (!static_key_enabled (&tp->key)) -static_key_slow_inc (&tp->key); +tracepoint_update_call (tp, tp_funcs); +static_key_enable (&tp->key); +release_probes (old);return 0 ; }
@@ -281,9 +296,11 @@ static int tracepoint_remove_func(struct tracepoint *tp,if (tp->unregfunc &&static_key_enabled (&tp->key)) tp->unregfunc (); -if (static_key_enabled (&tp->key)) -static_key_slow_dec (&tp->key); +static_key_disable (&tp->key); + }else { +tracepoint_update_call (tp, tp_funcs); } +rcu_assign_pointer (tp->funcs, tp_funcs);release_probes (old);return 0 ;
Peter Zijlstra
maintainerPATCH v7 18/18 x86/perf, static_call: Optimize x86_pmu methods
In reply to Peter ZijlstraReplace many of the indirect calls with static_call().
The average PMI time, as measured by perf_sample_event_took()*:
PRE: 3283.03 [ns] POST: 3145.12 [ns]
Which is a ~138 [ns] win per PMI, or a ~4.2% decrease.
[*] on an IVB-EP, using: 'perf record -a -e cycles -- make O=defconfig-build/ -j80'
--- arch/x86/events/core.c | 140 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 100 insertions(+), 40 deletions(-)
arch/x86/events/core.c +94 -40
@@ -28,6 +28,7 @@#include <linux/bitops.h> #include <linux/device.h> #include <linux/nospec.h> +#include <linux/static_call.h> #include <asm/apic.h> #include <asm/stacktrace.h>
@@ -52,6 +53,34 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpuDEFINE_STATIC_KEY_FALSE (rdpmc_never_available_key);DEFINE_STATIC_KEY_FALSE (rdpmc_always_available_key); +/* +* This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined +* from just a typename, as opposed to an actual function. +*/ +DEFINE_STATIC_CALL_NULL (x86_pmu_handle_irq, *x86_pmu.handle_irq); +DEFINE_STATIC_CALL_NULL (x86_pmu_disable_all, *x86_pmu.disable_all); +DEFINE_STATIC_CALL_NULL (x86_pmu_enable_all, *x86_pmu.enable_all); +DEFINE_STATIC_CALL_NULL (x86_pmu_enable, *x86_pmu.enable); +DEFINE_STATIC_CALL_NULL (x86_pmu_disable, *x86_pmu.disable); + +DEFINE_STATIC_CALL_NULL (x86_pmu_add, *x86_pmu.add); +DEFINE_STATIC_CALL_NULL (x86_pmu_del, *x86_pmu.del); +DEFINE_STATIC_CALL_NULL (x86_pmu_read, *x86_pmu.read); + +DEFINE_STATIC_CALL_NULL (x86_pmu_schedule_events, *x86_pmu.schedule_events); +DEFINE_STATIC_CALL_NULL (x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); +DEFINE_STATIC_CALL_NULL (x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); + +DEFINE_STATIC_CALL_NULL (x86_pmu_start_scheduling, *x86_pmu.start_scheduling); +DEFINE_STATIC_CALL_NULL (x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); +DEFINE_STATIC_CALL_NULL (x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); + +DEFINE_STATIC_CALL_NULL (x86_pmu_sched_task, *x86_pmu.sched_task); +DEFINE_STATIC_CALL_NULL (x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx); + +DEFINE_STATIC_CALL_NULL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DEFINE_STATIC_CALL_NULL (x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); +u64 __read_mostly hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ]
@@ -660,7 +695,7 @@ static void x86_pmu_disable(struct pmu * cpuc->enabled =0 ;barrier (); - x86_pmu.disable_all (); +static_call (x86_pmu_disable_all)(); }void x86_pmu_enable_all (int added)
@@ -907,8 +942,7 @@ int x86_schedule_events(struct cpu_hw_evif (cpuc->txn_flags &PERF_PMU_TXN_ADD ) n0 -= cpuc->n_txn; -if (x86_pmu.start_scheduling) - x86_pmu.start_scheduling (cpuc); +static_call_cond (x86_pmu_start_scheduling)(cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { c = cpuc->event_constraint[i];
@@ -925,7 +959,7 @@ int x86_schedule_events(struct cpu_hw_ev* change due to external factors (sibling state, allow_tfa). */ if (!c || (c->flags &PERF_X86_EVENT_DYNAMIC )) { - c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); + c =static_call (x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); cpuc->event_constraint[i] = c; }
@@ -1008,8 +1042,7 @@ int x86_schedule_events(struct cpu_hw_evif (!unsched && assign) {for (i =0 ; i < n; i++) { e = cpuc->event_list[i]; -if (x86_pmu.commit_scheduling) - x86_pmu.commit_scheduling (cpuc, i, assign[i]); +static_call_cond (x86_pmu_commit_scheduling)(cpuc, i, assign[i]); } }else {for (i = n0; i < n; i++) {
@@ -1018,15 +1051,13 @@ int x86_schedule_events(struct cpu_hw_ev/* * release events that failed scheduling */ -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, e); +static_call_cond (x86_pmu_put_event_constraints)(cpuc, e); cpuc->event_constraint[i] =NULL ; } } -if (x86_pmu.stop_scheduling) - x86_pmu.stop_scheduling (cpuc); +static_call_cond (x86_pmu_stop_scheduling)(cpuc);return unsched ? -EINVAL :0 ; }
@@ -1217,7 +1248,7 @@ static void x86_pmu_enable(struct pmu *p cpuc->enabled =1 ;barrier (); - x86_pmu.enable_all (added); +static_call (x86_pmu_enable_all)(added); } static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -1338,7 +1369,7 @@ static int x86_pmu_add(struct perf_eventif (cpuc->txn_flags &PERF_PMU_TXN_ADD ) goto done_collect; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret) goto out;/*
@@ -1356,13 +1387,11 @@ static int x86_pmu_add(struct perf_event cpuc->n_added += n - n0; cpuc->n_txn += n - n0; -if (x86_pmu.add) { -/* -* This is before x86_pmu_enable() will call x86_pmu_start(), -* so we enable LBRs before an event needs them etc.. -*/ - x86_pmu.add (event); - } +/* +* This is before x86_pmu_enable() will call x86_pmu_start(), +* so we enable LBRs before an event needs them etc.. +*/ +static_call_cond (x86_pmu_add)(event); ret =0 ; out:
@@ -1390,7 +1419,7 @@ static void x86_pmu_start(struct perf_ev cpuc->events[idx] = event;__set_bit (idx, cpuc->active_mask);__set_bit (idx, cpuc->running); - x86_pmu.enable (event); +static_call (x86_pmu_enable)(event);perf_event_update_userpage (event); }
@@ -1460,7 +1489,7 @@ void x86_pmu_stop(struct perf_event *evestruct hw_perf_event *hwc = &event->hw;if (test_bit (hwc->idx, cpuc->active_mask)) { - x86_pmu.disable (event); +static_call (x86_pmu_disable)(event);__clear_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] =NULL ;WARN_ON_ONCE (hwc->state &PERF_HES_STOPPED );
@@ -1510,8 +1539,7 @@ static void x86_pmu_del(struct perf_evenif (i >= cpuc->n_events - cpuc->n_added) --cpuc->n_added; -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, event); +static_call_cond (x86_pmu_put_event_constraints)(cpuc, event);/* Delete the array entry. */ while (++i < cpuc->n_events) {
@@ -1524,13 +1552,12 @@ static void x86_pmu_del(struct perf_evenperf_event_update_userpage (event); do_del: -if (x86_pmu.del) { -/* -* This is after x86_pmu_stop(); so we disable LBRs after any -* event can need them etc.. -*/ - x86_pmu.del (event); - } + +/* +* This is after x86_pmu_stop(); so we disable LBRs after any +* event can need them etc.. +*/ +static_call_cond (x86_pmu_del)(event); }int x86_pmu_handle_irq (struct pt_regs *regs)
@@ -1608,7 +1635,7 @@ perf_event_nmi_handler(unsigned int cmd,return NMI_DONE ; start_clock =sched_clock (); - ret = x86_pmu.handle_irq (regs); + ret =static_call (x86_pmu_handle_irq)(regs); finish_clock =sched_clock ();perf_sample_event_took (finish_clock - start_clock);
@@ -1821,6 +1848,38 @@ ssize_t x86_event_sysfs_show(char *page,static struct attribute_group x86_pmu_attr_group;static struct attribute_group x86_pmu_caps_group; +static void x86_pmu_static_call_update (void ) +{ +static_call_update (x86_pmu_handle_irq, x86_pmu.handle_irq); +static_call_update (x86_pmu_disable_all, x86_pmu.disable_all); +static_call_update (x86_pmu_enable_all, x86_pmu.enable_all); +static_call_update (x86_pmu_enable, x86_pmu.enable); +static_call_update (x86_pmu_disable, x86_pmu.disable); + +static_call_update (x86_pmu_add, x86_pmu.add); +static_call_update (x86_pmu_del, x86_pmu.del); +static_call_update (x86_pmu_read, x86_pmu.read); + +static_call_update (x86_pmu_schedule_events, x86_pmu.schedule_events); +static_call_update (x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); +static_call_update (x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); + +static_call_update (x86_pmu_start_scheduling, x86_pmu.start_scheduling); +static_call_update (x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); +static_call_update (x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); + +static_call_update (x86_pmu_sched_task, x86_pmu.sched_task); +static_call_update (x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx); + +static_call_update (x86_pmu_drain_pebs, x86_pmu.drain_pebs); +static_call_update (x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); +} + +static void _x86_pmu_read (struct perf_event *event) +{ +x86_perf_event_update (event); +} +static int __init init_hw_perf_events (void ) {struct x86_pmu_quirk *quirk;
@@ -1889,6 +1948,11 @@ static int __init init_hw_perf_events(vopr_info ("... fixed-purpose events: %d\n" , x86_pmu.num_counters_fixed);pr_info ("... event mask: %016Lx\n" , x86_pmu.intel_ctrl); +if (!x86_pmu.read) + x86_pmu.read = _x86_pmu_read; + +x86_pmu_static_call_update (); +/* * Install callbacks. Core will call them for each online * cpu.
@@ -1925,11 +1989,9 @@ static int __init init_hw_perf_events(vo }early_initcall (init_hw_perf_events); -static inline void x86_pmu_read (struct perf_event *event) +static void x86_pmu_read (struct perf_event *event) { -if (x86_pmu.read) -return x86_pmu.read (event); -x86_perf_event_update (event); +static_call (x86_pmu_read)(event); }/*
@@ -2006,7 +2068,7 @@ static int x86_pmu_commit_txn(struct pmuif (!x86_pmu_initialized ())return -EAGAIN ; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret)return ret;
@@ -2299,15 +2361,13 @@ static const struct attribute_group *x86static void x86_pmu_sched_task (struct perf_event_context *ctx,bool sched_in) { -if (x86_pmu.sched_task) - x86_pmu.sched_task (ctx, sched_in); +static_call_cond (x86_pmu_sched_task)(ctx, sched_in); }static void x86_pmu_swap_task_ctx (struct perf_event_context *prev,struct perf_event_context *next) { -if (x86_pmu.swap_task_ctx) - x86_pmu.swap_task_ctx (prev, next); +static_call_cond (x86_pmu_swap_task_ctx)(prev, next); }void perf_check_microcode (void )
[tip: core/static_call] x86/perf, static_call: Optimize x86_pmu methods
The following commit has been merged into the core/static_call branch of tip:
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 7c9903c9bf716d89b34f96cc2ed64e28dabf570b Gitweb: https://git.kernel.org/tip/7c9903c9bf716d89b34f96cc2ed64e28dabf570b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 18 Aug 2020 15:57:53 +02:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
x86/perf, static_call: Optimize x86_pmu methods
Replace many of the indirect calls with static_call().
The average PMI time, as measured by perf_sample_event_took()*:
PRE: 3283.03 [ns]
POST: 3145.12 [ns]
Which is a ~138 [ns] win per PMI, or a ~4.2% decrease.
[*] on an IVB-EP, using: 'perf record -a -e cycles -- make O=defconfig-build/ -j80'
--- arch/x86/events/core.c | 134 ++++++++++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 40 deletions(-)
arch/x86/events/core.c +94 -40
@@ -28,6 +28,7 @@#include <linux/bitops.h> #include <linux/device.h> #include <linux/nospec.h> +#include <linux/static_call.h> #include <asm/apic.h> #include <asm/stacktrace.h>
@@ -52,6 +53,34 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {DEFINE_STATIC_KEY_FALSE (rdpmc_never_available_key);DEFINE_STATIC_KEY_FALSE (rdpmc_always_available_key); +/* +* This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined +* from just a typename, as opposed to an actual function. +*/ +DEFINE_STATIC_CALL_NULL (x86_pmu_handle_irq, *x86_pmu.handle_irq); +DEFINE_STATIC_CALL_NULL (x86_pmu_disable_all, *x86_pmu.disable_all); +DEFINE_STATIC_CALL_NULL (x86_pmu_enable_all, *x86_pmu.enable_all); +DEFINE_STATIC_CALL_NULL (x86_pmu_enable, *x86_pmu.enable); +DEFINE_STATIC_CALL_NULL (x86_pmu_disable, *x86_pmu.disable); + +DEFINE_STATIC_CALL_NULL (x86_pmu_add, *x86_pmu.add); +DEFINE_STATIC_CALL_NULL (x86_pmu_del, *x86_pmu.del); +DEFINE_STATIC_CALL_NULL (x86_pmu_read, *x86_pmu.read); + +DEFINE_STATIC_CALL_NULL (x86_pmu_schedule_events, *x86_pmu.schedule_events); +DEFINE_STATIC_CALL_NULL (x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); +DEFINE_STATIC_CALL_NULL (x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); + +DEFINE_STATIC_CALL_NULL (x86_pmu_start_scheduling, *x86_pmu.start_scheduling); +DEFINE_STATIC_CALL_NULL (x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); +DEFINE_STATIC_CALL_NULL (x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); + +DEFINE_STATIC_CALL_NULL (x86_pmu_sched_task, *x86_pmu.sched_task); +DEFINE_STATIC_CALL_NULL (x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx); + +DEFINE_STATIC_CALL_NULL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DEFINE_STATIC_CALL_NULL (x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); +u64 __read_mostly hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ]
@@ -660,7 +689,7 @@ static void x86_pmu_disable(struct pmu *pmu) cpuc->enabled =0 ;barrier (); - x86_pmu.disable_all (); +static_call (x86_pmu_disable_all)(); }void x86_pmu_enable_all (int added)
@@ -907,8 +936,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)if (cpuc->txn_flags &PERF_PMU_TXN_ADD ) n0 -= cpuc->n_txn; -if (x86_pmu.start_scheduling) - x86_pmu.start_scheduling (cpuc); +static_call_cond (x86_pmu_start_scheduling)(cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { c = cpuc->event_constraint[i];
@@ -925,7 +953,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)* change due to external factors (sibling state, allow_tfa). */ if (!c || (c->flags &PERF_X86_EVENT_DYNAMIC )) { - c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); + c =static_call (x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); cpuc->event_constraint[i] = c; }
@@ -1008,8 +1036,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)if (!unsched && assign) {for (i =0 ; i < n; i++) { e = cpuc->event_list[i]; -if (x86_pmu.commit_scheduling) - x86_pmu.commit_scheduling (cpuc, i, assign[i]); +static_call_cond (x86_pmu_commit_scheduling)(cpuc, i, assign[i]); } }else {for (i = n0; i < n; i++) {
@@ -1018,15 +1045,13 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)/* * release events that failed scheduling */ -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, e); +static_call_cond (x86_pmu_put_event_constraints)(cpuc, e); cpuc->event_constraint[i] =NULL ; } } -if (x86_pmu.stop_scheduling) - x86_pmu.stop_scheduling (cpuc); +static_call_cond (x86_pmu_stop_scheduling)(cpuc);return unsched ? -EINVAL :0 ; }
@@ -1226,7 +1251,7 @@ static void x86_pmu_enable(struct pmu *pmu) cpuc->enabled =1 ;barrier (); - x86_pmu.enable_all (added); +static_call (x86_pmu_enable_all)(added); } static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -1347,7 +1372,7 @@ static int x86_pmu_add(struct perf_event *event, int flags)if (cpuc->txn_flags &PERF_PMU_TXN_ADD ) goto done_collect; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret) goto out;/*
@@ -1365,13 +1390,11 @@ done_collect: cpuc->n_added += n - n0; cpuc->n_txn += n - n0; -if (x86_pmu.add) { -/* -* This is before x86_pmu_enable() will call x86_pmu_start(), -* so we enable LBRs before an event needs them etc.. -*/ - x86_pmu.add (event); - } +/* +* This is before x86_pmu_enable() will call x86_pmu_start(), +* so we enable LBRs before an event needs them etc.. +*/ +static_call_cond (x86_pmu_add)(event); ret =0 ; out:
@@ -1399,7 +1422,7 @@ static void x86_pmu_start(struct perf_event *event, int flags) cpuc->events[idx] = event;__set_bit (idx, cpuc->active_mask);__set_bit (idx, cpuc->running); - x86_pmu.enable (event); +static_call (x86_pmu_enable)(event);perf_event_update_userpage (event); }
@@ -1469,7 +1492,7 @@ void x86_pmu_stop(struct perf_event *event, int flags)struct hw_perf_event *hwc = &event->hw;if (test_bit (hwc->idx, cpuc->active_mask)) { - x86_pmu.disable (event); +static_call (x86_pmu_disable)(event);__clear_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] =NULL ;WARN_ON_ONCE (hwc->state &PERF_HES_STOPPED );
@@ -1519,8 +1542,7 @@ static void x86_pmu_del(struct perf_event *event, int flags)if (i >= cpuc->n_events - cpuc->n_added) --cpuc->n_added; -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, event); +static_call_cond (x86_pmu_put_event_constraints)(cpuc, event);/* Delete the array entry. */ while (++i < cpuc->n_events) {
@@ -1533,13 +1555,12 @@ static void x86_pmu_del(struct perf_event *event, int flags)perf_event_update_userpage (event); do_del: -if (x86_pmu.del) { -/* -* This is after x86_pmu_stop(); so we disable LBRs after any -* event can need them etc.. -*/ - x86_pmu.del (event); - } + +/* +* This is after x86_pmu_stop(); so we disable LBRs after any +* event can need them etc.. +*/ +static_call_cond (x86_pmu_del)(event); }int x86_pmu_handle_irq (struct pt_regs *regs)
@@ -1617,7 +1638,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)return NMI_DONE ; start_clock =sched_clock (); - ret = x86_pmu.handle_irq (regs); + ret =static_call (x86_pmu_handle_irq)(regs); finish_clock =sched_clock ();perf_sample_event_took (finish_clock - start_clock);
@@ -1830,6 +1851,38 @@ ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)static struct attribute_group x86_pmu_attr_group;static struct attribute_group x86_pmu_caps_group; +static void x86_pmu_static_call_update (void ) +{ +static_call_update (x86_pmu_handle_irq, x86_pmu.handle_irq); +static_call_update (x86_pmu_disable_all, x86_pmu.disable_all); +static_call_update (x86_pmu_enable_all, x86_pmu.enable_all); +static_call_update (x86_pmu_enable, x86_pmu.enable); +static_call_update (x86_pmu_disable, x86_pmu.disable); + +static_call_update (x86_pmu_add, x86_pmu.add); +static_call_update (x86_pmu_del, x86_pmu.del); +static_call_update (x86_pmu_read, x86_pmu.read); + +static_call_update (x86_pmu_schedule_events, x86_pmu.schedule_events); +static_call_update (x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); +static_call_update (x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); + +static_call_update (x86_pmu_start_scheduling, x86_pmu.start_scheduling); +static_call_update (x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); +static_call_update (x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); + +static_call_update (x86_pmu_sched_task, x86_pmu.sched_task); +static_call_update (x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx); + +static_call_update (x86_pmu_drain_pebs, x86_pmu.drain_pebs); +static_call_update (x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); +} + +static void _x86_pmu_read (struct perf_event *event) +{ +x86_perf_event_update (event); +} +static int __init init_hw_perf_events (void ) {struct x86_pmu_quirk *quirk;
@@ -1898,6 +1951,11 @@ static int __init init_hw_perf_events(void)pr_info ("... fixed-purpose events: %d\n" , x86_pmu.num_counters_fixed);pr_info ("... event mask: %016Lx\n" , x86_pmu.intel_ctrl); +if (!x86_pmu.read) + x86_pmu.read = _x86_pmu_read; + +x86_pmu_static_call_update (); +/* * Install callbacks. Core will call them for each online * cpu.
@@ -1934,11 +1992,9 @@ out: }early_initcall (init_hw_perf_events); -static inline void x86_pmu_read (struct perf_event *event) +static void x86_pmu_read (struct perf_event *event) { -if (x86_pmu.read) -return x86_pmu.read (event); -x86_perf_event_update (event); +static_call (x86_pmu_read)(event); }/*
@@ -2015,7 +2071,7 @@ static int x86_pmu_commit_txn(struct pmu *pmu)if (!x86_pmu_initialized ())return -EAGAIN ; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret)return ret;
@@ -2308,15 +2364,13 @@ static const struct attribute_group *x86_pmu_attr_groups[] = {static void x86_pmu_sched_task (struct perf_event_context *ctx,bool sched_in) { -if (x86_pmu.sched_task) - x86_pmu.sched_task (ctx, sched_in); +static_call_cond (x86_pmu_sched_task)(ctx, sched_in); }static void x86_pmu_swap_task_ctx (struct perf_event_context *prev,struct perf_event_context *next) { -if (x86_pmu.swap_task_ctx) - x86_pmu.swap_task_ctx (prev, next); +static_call_cond (x86_pmu_swap_task_ctx)(prev, next); }void perf_check_microcode (void )