ftrace: 3.18 queue
Steven Rostedt
maintainercover ftrace: 3.18 queue
This is my current queue for 3.18. The main update is a rewrite of the trampoline accounting to get rid of the large memory usage needed when having a trampoline for all functions (the current method creates a descriptor for each function). The new way uses the current logic of the hash tables, where an empty hash can mean all functions and also the use of the notrace hash for all but a few functions.
OFF TOPIC:
As some of you may know, I've recently been having chronic pain in my upper back, shoulder and down the arm. This has not let up, although the narcotic drugs and steroids I've been prescribed has helped a lot. Technically, I'm not suppose to be using a computer at all. But I couldn't help myself and spent a few minutes here and there working on getting these patches to pass all my tests. Note that I usually paid for that work with an hour or so of more pain down the arm. There is a good chance I will be having spinal surgery in the near future to remove the problem cervical discs and have those vertebra fused. Hopefully, I will be back to 100% shortly afterward. But until then, I will still be on hiatus from the community again. But hopefully not for too long.
git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
for-next
Head SHA1: fb5a613b4f310d6d520daf295547ab35b0ac58a3
Andreea-Cristina Bernat (1):
kernel: trace_syscalls: Replace rcu_assign_pointer() with RCU_INIT_POINTER()
Dave Hansen (1):
tracing: generate RCU warnings even when tracepoints are disabled
Steven Rostedt (Red Hat) (7):
ftrace: Add separate function for non recursive callbacks
ftrace: Add helper function ftrace_ops_get_func()
ftrace: Set callback to ftrace_stub when no ops are registered
ftrace: Remove freeing of old_hash from ftrace_hash_move()
ftrace: Grab any ops for a rec for enabled_functions output
ftrace: Annotate the ops operation on update
ftrace: Replace tramp_hash with old_*_hash to save space
----
include/linux/ftrace.h | 10 +- include/linux/tracepoint.h | 11 ++ kernel/trace/ftrace.c | 401 ++++++++++++++++++++++++------------------ kernel/trace/trace_syscalls.c | 4 +- 4 files changed, 248 insertions(+), 178 deletions(-)
Steven Rostedt
maintainerPATCH 1/9 ftrace: Add separate function for non recursive callbacks
From: "Steven Rostedt (Red Hat)" <[email protected]>
Instead of using the generic list function for callbacks that are not recursive, call a new helper function from the mcount trampoline called ftrace_ops_recur_func() that will do the recursion checking for the callback.
This eliminates an indirection as well as will help in future code that will use dynamically allocated trampolines.
--- kernel/trace/ftrace.c | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-)
kernel/trace/ftrace.c +31 -2
@@ -113,6 +113,9 @@ ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;static struct ftrace_ops global_ops;static struct ftrace_ops control_ops; +static void ftrace_ops_recurs_func (unsigned long ip,unsigned long parent_ip, +struct ftrace_ops *op,struct pt_regs *regs); +#if ARCH_SUPPORTS_FTRACE_OPS static void ftrace_ops_list_func (unsigned long ip,unsigned long parent_ip,struct ftrace_ops *op,struct pt_regs *regs);
@@ -258,11 +261,18 @@ static void update_ftrace_function(void)if (ftrace_ops_list == &ftrace_list_end || (ftrace_ops_list->next == &ftrace_list_end && !(ftrace_ops_list->flags &FTRACE_OPS_FL_DYNAMIC ) && - (ftrace_ops_list->flags &FTRACE_OPS_FL_RECURSION_SAFE ) && !FTRACE_FORCE_LIST_FUNC )) {/* Set the ftrace_ops that the arch callback uses */ set_function_trace_op = ftrace_ops_list; - func = ftrace_ops_list->func; +/* +* If the func handles its own recursion, call it directly. +* Otherwise call the recursion protected function that +* will call the ftrace ops function. +*/ +if (ftrace_ops_list->flags &FTRACE_OPS_FL_RECURSION_SAFE ) + func = ftrace_ops_list->func; +else + func = ftrace_ops_recurs_func; }else {/* Just use the default ftrace_ops */ set_function_trace_op = &ftrace_list_end;
@@ -4827,6 +4837,25 @@ static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip) }#endif +/* +* If there's only one function registered but it does not support +* recursion, this function will be called by the mcount trampoline. +* This function will handle recursion protection. +*/ +static void ftrace_ops_recurs_func (unsigned long ip,unsigned long parent_ip, +struct ftrace_ops *op,struct pt_regs *regs) +{ +int bit; + + bit =trace_test_and_set_recursion (TRACE_LIST_START ,TRACE_LIST_MAX ); +if (bit <0 ) +return ; + + op->func (ip, parent_ip, op, regs); + +trace_clear_recursion (bit); +} +static void clear_ftrace_swapper (void ) {struct task_struct *p;
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 2/9 ftrace: Add helper function ftrace_ops_get_func()
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
Add the helper function to what the mcount trampoline is to call for a ftrace_ops function. This helper will be used by arch code in the future to set up dynamic trampolines. But as this does the same tests that are performed in choosing what function to call for the default mcount trampoline, might as well use it to clean up the existing code.
--- include/linux/ftrace.h | 2 ++ kernel/trace/ftrace.c | 47 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 12 deletions(-)
include/linux/ftrace.h +2 -0
@@ -56,6 +56,8 @@ struct ftrace_ops;typedef void (*ftrace_func_t )(unsigned long ip,unsigned long parent_ip,struct ftrace_ops *op,struct pt_regs *regs); +ftrace_func_t ftrace_ops_get_func (struct ftrace_ops *ops); +/* * FTRACE_OPS_FL_* bits denote the state of ftrace_ops struct and are * set in the flags member.
kernel/trace/ftrace.c +35 -12
61 lines not on this page · raw message
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 3/9 ftrace: Set callback to ftrace_stub when no ops are registered
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
The clean up that adds the helper function ftrace_ops_get_func() caused the default function to not change when DYNAMIC_FTRACE was not set and no ftrace_ops were registered. Although static tracing is not very useful (not having DYNAMIC_FTRACE set), it is still supported and we don't want to break it.
Clean up the if statement even more to specifically have the default function call ftrace_stub when no ftrace_ops are registered. This fixes the small bug for static tracing as well as makes the code a bit more understandable.
--- kernel/trace/ftrace.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-)
kernel/trace/ftrace.c +13 -6
@@ -254,17 +254,24 @@ static void update_ftrace_function(void)ftrace_func_t func;/* +* Prepare the ftrace_ops that the arch callback will use. +* If there's only one ftrace_ops registered, the ftrace_ops_list +* will point to the ops we want. +*/ + set_function_trace_op = ftrace_ops_list; + +/* If there's no ftrace_ops registered, just call the stub function */ +if (ftrace_ops_list == &ftrace_list_end) { + func = ftrace_stub; + +/* * If we are at the end of the list and this ops is * recursion safe and not dynamic and the arch supports passing ops, * then have the mcount trampoline call the function directly. */ -if (ftrace_ops_list == &ftrace_list_end || - (ftrace_ops_list->next == &ftrace_list_end)) { - -/* Set the ftrace_ops that the arch callback uses */ - set_function_trace_op = ftrace_ops_list; - + }else if (ftrace_ops_list->next == &ftrace_list_end) { func =ftrace_ops_get_func (ftrace_ops_list); + }else {/* Just use the default ftrace_ops */ set_function_trace_op = &ftrace_list_end;
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 4/9 ftrace: Remove freeing of old_hash from ftrace_hash_move()
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
ftrace_hash_move() currently frees the old hash that is passed to it after replacing the pointer with the new hash. Instead of having the function do that chore, have the caller perform the free.
This lets the ftrace_hash_move() be used a bit more freely, which is needed for changing the way the trampoline logic is done.
--- kernel/trace/ftrace.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-)
kernel/trace/ftrace.c +21 -10
@@ -1316,7 +1316,6 @@ ftrace_hash_move(struct ftrace_ops *ops, int enable,struct ftrace_func_entry *entry;struct hlist_node *tn;struct hlist_head *hhd; -struct ftrace_hash *old_hash;struct ftrace_hash *new_hash;int size = src->count;int bits =0 ;
@@ -1361,9 +1360,7 @@ update:*/ ftrace_hash_rec_disable_modify (ops, enable); - old_hash = *dst;rcu_assign_pointer (*dst, new_hash); -free_ftrace_hash_rcu (old_hash);ftrace_hash_rec_enable_modify (ops, enable);
@@ -3408,6 +3405,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, {struct ftrace_func_probe *entry;struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; +struct ftrace_hash *old_hash = *orig_hash;struct ftrace_hash *hash;struct ftrace_page *pg;struct dyn_ftrace *rec;
@@ -3426,7 +3424,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,mutex_lock (&trace_probe_ops.func_hash->regex_lock); - hash =alloc_and_copy_ftrace_hash (FTRACE_HASH_DEFAULT_BITS , *orig_hash); + hash =alloc_and_copy_ftrace_hash (FTRACE_HASH_DEFAULT_BITS , old_hash);if (!hash) { count = -ENOMEM ; goto out;
@@ -3485,7 +3483,9 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, }while_for_each_ftrace_rec (); ret =ftrace_hash_move (&trace_probe_ops,1 , orig_hash, hash); -if (ret <0 ) +if (!ret) +free_ftrace_hash_rcu (old_hash); +else count = ret;__enable_ftrace_function_probe ();
@@ -3512,6 +3512,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,struct ftrace_func_probe *entry;struct ftrace_func_probe *p;struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash; +struct ftrace_hash *old_hash = *orig_hash;struct list_head free_list;struct ftrace_hash *hash;struct hlist_node *tmp;
@@ -3519,6 +3520,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,int type =MATCH_FULL ;int i, len =0 ;char *search; +int ret;if (glob && (strcmp (glob,"*" ) ==0 || !strlen (glob))) glob =NULL ;
@@ -3577,8 +3579,11 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,* Remove after the disable is called. Otherwise, if the last * probe is removed, a null hash means *all enabled*. */ -ftrace_hash_move (&trace_probe_ops,1 , orig_hash, hash); + ret =ftrace_hash_move (&trace_probe_ops,1 , orig_hash, hash);synchronize_sched (); +if (!ret) +free_ftrace_hash_rcu (old_hash); +list_for_each_entry_safe (entry, p, &free_list, free_list) {list_del (&entry->free_list);ftrace_free_entry (entry);
@@ -3776,6 +3781,7 @@ ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, unsigned long ip, int remove, int reset, int enable) { struct ftrace_hash **orig_hash; +struct ftrace_hash *old_hash;struct ftrace_hash *hash;int ret;
@@ -3810,10 +3816,12 @@ ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, }mutex_lock (&ftrace_lock); + old_hash = *orig_hash; ret =ftrace_hash_move (ops, enable, orig_hash, hash); -if (!ret) +if (!ret) {ftrace_ops_update_code (ops); - +free_ftrace_hash_rcu (old_hash); + }mutex_unlock (&ftrace_lock); out_regex_unlock:
@@ -4022,6 +4030,7 @@ int ftrace_regex_release(struct inode *inode, struct file *file)struct seq_file *m = (struct seq_file *)file->private_data;struct ftrace_iterator *iter;struct ftrace_hash **orig_hash; +struct ftrace_hash *old_hash;struct trace_parser *parser;int filter_hash;int ret;
@@ -4051,11 +4060,13 @@ int ftrace_regex_release(struct inode *inode, struct file *file) orig_hash = &iter->ops->func_hash->notrace_hash;mutex_lock (&ftrace_lock); + old_hash = *orig_hash; ret =ftrace_hash_move (iter->ops, filter_hash, orig_hash, iter->hash); -if (!ret) +if (!ret) {ftrace_ops_update_code (iter->ops); - +free_ftrace_hash_rcu (old_hash); + }mutex_unlock (&ftrace_lock); }
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 5/9 ftrace: Grab any ops for a rec for enabled_functions output
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
When dumping the enabled_functions, use the first op that is found with a trampoline to the record, as there should only be one, as only one ops can be registered to a function that has a trampoline.
--- kernel/trace/ftrace.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-)
kernel/trace/ftrace.c +20 -1
@@ -1901,6 +1901,25 @@ int ftrace_test_record(struct dyn_ftrace *rec, int enable) }static struct ftrace_ops * +ftrace_find_tramp_ops_any (struct dyn_ftrace *rec) +{ +struct ftrace_ops *op; + +do_for_each_ftrace_op (op, ftrace_ops_list) { + +if (!op->trampoline) +continue ; + +if (ftrace_lookup_ip (op->func_hash->filter_hash, rec->ip) && + (ftrace_hash_empty (op->func_hash->notrace_hash) || + !ftrace_lookup_ip (op->func_hash->notrace_hash, rec->ip))) +return op; + }while_for_each_ftrace_op (op); + +return NULL ; +} + +static struct ftrace_ops *ftrace_find_tramp_ops_curr (struct dyn_ftrace *rec) {struct ftrace_ops *op;
@@ -2966,7 +2985,7 @@ static int t_show(struct seq_file *m, void *v)if (rec->flags &FTRACE_FL_TRAMP_EN ) {struct ftrace_ops *ops; - ops =ftrace_find_tramp_ops_curr (rec); + ops =ftrace_find_tramp_ops_any (rec);if (ops && ops->trampoline)seq_printf (m,"\ttramp: %pS" , (void *)ops->trampoline);
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 6/9 ftrace: Annotate the ops operation on update
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
Add three new flags for ftrace_ops:
FTRACE_OPS_FL_ADDING
FTRACE_OPS_FL_REMOVING
FTRACE_OPS_FL_MODIFYING
These will be set for the ftrace_ops when they are first added to the function tracing, being removed from function tracing or just having their functions changed from function tracing, respectively.
This will be needed to remove the tramp_hash, which can grow quite big. The tramp_hash is used to note what functions a ftrace_ops is using a trampoline for. Denoting which ftrace_ops is being modified, will allow us to use the ftrace_ops hashes themselves, which are much smaller as they have a global flag to denote if a ftrace_ops is tracing all functions, as well as a notrace hash if the ftrace_ops is tracing all but a few. The tramp_hash just creates a hash item for every function, which can go into the 10s of thousands if all functions are using the ftrace_ops trampoline.
--- include/linux/ftrace.h | 6 ++++++ kernel/trace/ftrace.c | 45 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 6 deletions(-)
include/linux/ftrace.h +6 -0
@@ -91,6 +91,9 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops);* INITIALIZED - The ftrace_ops has already been initialized (first use time * register_ftrace_function() is called, it will initialized the ops) * DELETED - The ops are being deleted, do not let them be registered again. +* ADDING - The ops is in the process of being added. +* REMOVING - The ops is in the process of being removed. +* MODIFYING - The ops is in the process of changing its filter functions. */ enum {FTRACE_OPS_FL_ENABLED =1 <<0 ,
@@ -102,6 +105,9 @@ enum {FTRACE_OPS_FL_STUB =1 <<6 ,FTRACE_OPS_FL_INITIALIZED =1 <<7 ,FTRACE_OPS_FL_DELETED =1 <<8 , +FTRACE_OPS_FL_ADDING =1 <<9 , +FTRACE_OPS_FL_REMOVING =1 <<10 , + FTRACE_OPS_FL_MODIFYING = 1 << 11, };#ifdef CONFIG_DYNAMIC_FTRACE
kernel/trace/ftrace.c +39 -6
114 lines not on this page · raw message
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 7/9 ftrace: Replace tramp_hash with old_*_hash to save space
In reply to Steven RostedtFrom: "Steven Rostedt (Red Hat)" <[email protected]>
Allowing function callbacks to declare their own trampolines requires that each ftrace_ops that has a trampoline must have some sort of accounting that keeps track of which ops has a trampoline attached to a record.
The easy way to solve this was to add a "tramp_hash" that created a hash entry for every function that a ops uses with a trampoline. But since we can have literally tens of thousands of functions being traced, that means we need tens of thousands of descriptors to map the ops to the function in the hash. This is quite expensive and can cause enabling and disabling the function graph tracer to take some time to start and stop. It can take up to several seconds to disable or enable all functions in the function graph tracer for this reason.
The better approach albeit more complex, is to keep track of how ops are being enabled and disabled, and use that along with the counting of the number of ops attached to records, to determive what ops has a trampoline attached to a record at enabling and disabling of tracing.
To do this, the tramp_hash has been replaced with an old_filter_hash and old_notrace_hash, which get the copy of the ops filter_hash and notrace_hash respectively. The old hashes is kept until the ops has been modified or removed and the old hashes are used with the logic of the accounting to determine the ops that have the trampoline of a record. The reason this has less of a footprint is due to the trick that an "empty" hash in the filter_hash means "all functions" and an empty hash in the notrace hash means "no functions" in the hash.
This is much more efficienct, doesn't have the delay, and takes up much less memory, as we do not need to map all the functions but just figure out which functions are mapped at the time it is enabled or disabled.
--- include/linux/ftrace.h | 2 +- kernel/trace/ftrace.c | 239 +++++++++++++++++-------------------------------- 2 files changed, 85 insertions(+), 156 deletions(-)
include/linux/ftrace.h +1 -1
@@ -140,7 +140,7 @@ struct ftrace_ops {int nr_trampolines;struct ftrace_ops_hash local_hash;struct ftrace_ops_hash *func_hash; -struct ftrace_hash *tramp_hash; +struct ftrace_ops_hash old_hash;unsigned long trampoline;#endif };
kernel/trace/ftrace.c +84 -155
349 lines not on this page · raw message
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 8/9 tracing: generate RCU warnings even when tracepoints are disabled
In reply to Steven RostedtFrom: Dave Hansen <[email protected]>
Dave Jones reported seeing a bug from one of my TLB tracepoints:
http://lkml.kernel.org/r/[email protected]
I've been running these patches for months and never saw this. But, a big chunk of my testing, especially with all the debugging enabled, was in a vm where intel_idle doesn't work. On the systems where I was using intel_idle, I never had lockdep enabled and this tracepoint on at the same time.
This patch ensures that whenever we have lockdep available, we do _some_ RCU activity at the site of the tracepoint, despite whether the tracepoint's condition matches or even if the tracepoint itself is completely disabled. This is a bit of a hack, but it is pretty self-contained.
I confirmed that with this patch plus lockdep I get the same splat as Dave Jones did, but without enabling the tracepoint explicitly.
--- include/linux/tracepoint.h | 11 +++++++++++ 1 file changed, 11 insertions(+)
include/linux/tracepoint.h +11 -0
@@ -157,6 +157,12 @@ extern void syscall_unregfunc(void);* Make sure the alignment of the structure in the __tracepoints section will * not add unwanted padding between the beginning of the section and the * structure. Force alignment to the same alignment as the section start. +* +* When lockdep is enabled, we make sure to always do the RCU portions of +* the tracepoint code, regardless of whether tracing is on or we match the +* condition. This lets us find RCU issues triggered with tracepoints even +* when this tracepoint is off. This code has no purpose other than poking +* RCU a bit. */ #define __DECLARE_TRACE (name, proto, args, cond, data_proto, data_args) \ extern struct tracepoint __tracepoint_##name; \
@@ -167,6 +173,11 @@ extern void syscall_unregfunc(void);TP_PROTO (data_proto), \TP_ARGS (data_args), \ TP_CONDITION(cond),,); \ +if (IS_ENABLED (CONFIG_LOCKDEP )) { \ +rcu_read_lock_sched_notrace (); \ + rcu_dereference_sched(__tracepoint_##name.funcs);\ +rcu_read_unlock_sched_notrace (); \ + } \ } \__DECLARE_TRACE_RCU (name,PARAMS (proto),PARAMS (args), \PARAMS (cond),PARAMS (data_proto),PARAMS (data_args)) \
Signature · 1 line
2.0.1
Steven Rostedt
maintainerPATCH 9/9 kernel: trace_syscalls: Replace rcu_assign_pointer() with RCU_INIT_POINTER()
In reply to Steven RostedtFrom: Andreea-Cristina Bernat <[email protected]>
The uses of "rcu_assign_pointer()" are NULLing out the pointers. According to RCU_INIT_POINTER()'s block comment:
"1. This use of RCU_INIT_POINTER() is NULLing out the pointer"
it is better to use it instead of rcu_assign_pointer() because it has a smaller overhead.
The following Coccinelle semantic patch was used:
@@
@@
- rcu_assign_pointer
+ RCU_INIT_POINTER
(..., NULL)
--- kernel/trace/trace_syscalls.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/trace/trace_syscalls.c +2 -2
@@ -425,7 +425,7 @@ static void unreg_event_syscall_enter(struct ftrace_event_file *file,return ;mutex_lock (&syscall_trace_lock); tr->sys_refcount_enter--; -rcu_assign_pointer (tr->enter_syscall_files[num],NULL ); +RCU_INIT_POINTER (tr->enter_syscall_files[num],NULL );if (!tr->sys_refcount_enter)unregister_trace_sys_enter (ftrace_syscall_enter, tr);mutex_unlock (&syscall_trace_lock);
@@ -463,7 +463,7 @@ static void unreg_event_syscall_exit(struct ftrace_event_file *file,return ;mutex_lock (&syscall_trace_lock); tr->sys_refcount_exit--; -rcu_assign_pointer (tr->exit_syscall_files[num],NULL ); +RCU_INIT_POINTER (tr->exit_syscall_files[num],NULL );if (!tr->sys_refcount_exit)unregister_trace_sys_exit (ftrace_syscall_exit, tr);mutex_unlock (&syscall_trace_lock);
Signature · 1 line
2.0.1