ksym_tracer: Eliminate trace concatenation and machine stall issues post removal
Subsystems (2)
A message that is not in the archive (sent off-list or not yet received): 20090619172035.443923337@prasadkr_t60p.in.ibm.com
PATCH 1/3 ksym_tracer: Eliminate trace concatenation and machine stall issues post removal
This patch fixes two issues reported with ksym tracer, seen after removal of a symbol from the tracer i) Concatenation of trace logs into a single line ii) Machine stall seen when logs are viewed using 'trace_pipe'.
Known issue: Upon removal, 'cat trace_pipe' (without any preceding command and any further output in the trace buffer) responds to SIGTERM quickly but only after an indeterminate amount of delay for SIGINT.
--- kernel/trace/trace.h | 12 +++++++++--- kernel/trace/trace_ksym.c | 11 ++++------- 2 files changed, 13 insertions(+), 10 deletions(-)
kernel/trace/trace_ksym.c +4 -7
@@ -71,7 +71,7 @@ void ksym_hbp_handler(struct hw_breakpoi {struct ring_buffer_event *event;struct trace_array *tr; -struct trace_ksym *entry; +struct trace_ksym_rb *entry;int pc;if (!ksym_tracing_enabled)
@@ -87,7 +87,7 @@ void ksym_hbp_handler(struct hw_breakpoi entry =ring_buffer_event_data (event);strlcpy (entry->ksym_name, hbp->info.name,KSYM_SYMBOL_LEN ); - entry->ksym_hbp = hbp; +memcpy (&(entry->ksym_hbp), hbp,sizeof (struct hw_breakpoint )); entry->ip =instruction_pointer (regs);strlcpy (entry->p_name, current->comm,TASK_COMM_LEN );#ifdef CONFIG_PROFILE_KSYM_TRACER
@@ -380,7 +380,7 @@ static enum print_line_t ksym_trace_outp {struct trace_entry *entry = iter->ent;struct trace_seq *s = &iter->seq; -struct trace_ksym *field; +struct trace_ksym_rb *field;char str[KSYM_SYMBOL_LEN ];int ret;
@@ -394,17 +394,14 @@ static enum print_line_t ksym_trace_outpif (!ret)return TRACE_TYPE_PARTIAL_LINE ; -switch (field->ksym_hbp->info.type) { +switch (field->ksym_hbp.info.type) {case HW_BREAKPOINT_WRITE : ret =trace_seq_printf (s," W " );break ;case HW_BREAKPOINT_RW : ret =trace_seq_printf (s," RW " );break ; -default : -return TRACE_TYPE_PARTIAL_LINE ; } -if (!ret)return TRACE_TYPE_PARTIAL_LINE ;
kernel/trace/trace.h +9 -3
@@ -216,15 +216,21 @@ struct syscall_trace_exit {extern int process_new_ksym_entry (char *ksymname,int op,unsigned long addr);struct trace_ksym { -struct trace_entry ent;struct hw_breakpoint *ksym_hbp;unsigned long ksym_addr; -unsigned long ip;#ifdef CONFIG_PROFILE_KSYM_TRACER unsigned long counter;#endif struct hlist_node ksym_hlist;char ksym_name[KSYM_NAME_LEN ]; +}; + +/* Ring buffer's copy of the breakpoint data */ +struct trace_ksym_rb { +struct trace_entry ent; +struct hw_breakpoint ksym_hbp; +unsigned long ip; +char ksym_name[KSYM_NAME_LEN ];char p_name[TASK_COMM_LEN ]; };
@@ -343,7 +349,7 @@ extern void __ftrace_bad_type(void); TRACE_SYSCALL_ENTER); \ IF_ASSIGN(var, ent, struct syscall_trace_exit, \TRACE_SYSCALL_EXIT ); \ - IF_ASSIGN(var, ent, struct trace_ksym, TRACE_KSYM); \ + IF_ASSIGN(var, ent, struct trace_ksym_rb, TRACE_KSYM); \__ftrace_bad_type (); \ }while (0 )
Quoted 30 lines · K.Prasad
This patch fixes two issues reported with ksym tracer, seen after removal of a symbol from the tracer i) Concatenation of trace logs into a single line ii) Machine stall seen when logs are viewed using '
trace_pipe'.Known issue: Upon removal, 'cat
trace_pipe' (without any preceding command and any further output in the trace buffer) responds to SIGTERM quickly but only after an indeterminate amount of delay for SIGINT.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace.h | 12 +++++++++--- kernel/trace/trace_ksym.c | 11 ++++------- 2 files changed, 13 insertions(+), 10 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +1
-1@@ -71,7 +71,7 @@ void ksym_hbp_handler(struct hw_breakpoi {struct ring_buffer_event *event;struct trace_array *tr; -struct trace_ksym *entry; +struct trace_ksym_rb *entry;int pc;if (!ksym_tracing_enabled)
kernel/trace/trace_ksym.c void ksym_hbp_handler(struct hw_breakpoi · line 90
Quoted 5 lines
kernel/trace/trace_ksym.c +1
-1@@ -87,7 +87,7 @@ void ksym_hbp_handler(struct hw_breakpoi entry =ring_buffer_event_data (event);strlcpy (entry->ksym_name, hbp->info.name,KSYM_SYMBOL_LEN ); - entry->ksym_hbp = hbp; +memcpy (&(entry->ksym_hbp), hbp,sizeof (struct hw_breakpoint ));
That looks wasteful. You only need the type from the arch_hw_breakpoint and there you copy the whole generic breakpoint.
Frederic.
Quoted 69 lines
kernel/trace/trace_ksym.c +2
-5entry->ip =instruction_pointer (regs);strlcpy (entry->p_name, current->comm,TASK_COMM_LEN );#ifdef CONFIG_PROFILE_KSYM_TRACER @@ -380,7 +380,7 @@ static enum print_line_t ksym_trace_outp {struct trace_entry *entry = iter->ent;struct trace_seq *s = &iter->seq; -struct trace_ksym *field; +struct trace_ksym_rb *field;char str[KSYM_SYMBOL_LEN ];int ret;@@ -394,17 +394,14 @@ static enum print_line_t ksym_trace_outpif (!ret)return TRACE_TYPE_PARTIAL_LINE ; -switch (field->ksym_hbp->info.type) { +switch (field->ksym_hbp.info.type) {case HW_BREAKPOINT_WRITE : ret =trace_seq_printf (s," W " );break ;case HW_BREAKPOINT_RW : ret =trace_seq_printf (s," RW " );break ; -default : -return TRACE_TYPE_PARTIAL_LINE ; } -if (!ret)return TRACE_TYPE_PARTIAL_LINE ;Index:
linux-2.6-tip.hbkpt/kernel/trace/trace.h
===================================================================kernel/trace/trace.h +9
-3@@ -216,15 +216,21 @@ struct syscall_trace_exit {extern int process_new_ksym_entry (char *ksymname,int op,unsigned long addr);struct trace_ksym { -struct trace_entry ent;struct hw_breakpoint *ksym_hbp;unsigned long ksym_addr; -unsigned long ip;#ifdef CONFIG_PROFILE_KSYM_TRACER unsigned long counter;#endif struct hlist_node ksym_hlist;char ksym_name[KSYM_NAME_LEN ]; +}; + +/* Ring buffer's copy of the breakpoint data */ +struct trace_ksym_rb { +struct trace_entry ent; +struct hw_breakpoint ksym_hbp; +unsigned long ip; +char ksym_name[KSYM_NAME_LEN ];char p_name[TASK_COMM_LEN ]; };@@ -343,7 +349,7 @@ extern void __ftrace_bad_type(void); TRACE_SYSCALL_ENTER); \ IF_ASSIGN(var, ent, struct syscall_trace_exit, \TRACE_SYSCALL_EXIT ); \ - IF_ASSIGN(var, ent, struct trace_ksym, TRACE_KSYM); \ + IF_ASSIGN(var, ent, struct trace_ksym_rb, TRACE_KSYM); \__ftrace_bad_type (); \ }while (0 )
Quoted 41 lines · Frederic Weisbecker
Quoted 30 lines · K.Prasad
This patch fixes two issues reported with ksym tracer, seen after removal of a symbol from the tracer i) Concatenation of trace logs into a single line ii) Machine stall seen when logs are viewed using '
trace_pipe'.Known issue: Upon removal, 'cat
trace_pipe' (without any preceding command and any further output in the trace buffer) responds to SIGTERM quickly but only after an indeterminate amount of delay for SIGINT.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace.h | 12 +++++++++--- kernel/trace/trace_ksym.c | 11 ++++------- 2 files changed, 13 insertions(+), 10 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +1
-1@@ -71,7 +71,7 @@ void ksym_hbp_handler(struct hw_breakpoi {struct ring_buffer_event *event;struct trace_array *tr; -struct trace_ksym *entry; +struct trace_ksym_rb *entry;int pc;if (!ksym_tracing_enabled)kernel/trace/trace_ksym.c void ksym_hbp_handler(struct hw_breakpoi · line 90
Quoted 5 lines
kernel/trace/trace_ksym.c +1
-1@@ -87,7 +87,7 @@ void ksym_hbp_handler(struct hw_breakpoi entry =ring_buffer_event_data (event);strlcpy (entry->ksym_name, hbp->info.name,KSYM_SYMBOL_LEN ); - entry->ksym_hbp = hbp; +memcpy (&(entry->ksym_hbp), hbp,sizeof (struct hw_breakpoint ));That looks wasteful. You only need the type from the
arch_hw_breakpointand there you copy the whole generic breakpoint.
Quoted 2 lines
Frederic.
Well, the entire copy operation for ksym_name and p_name is indeed a much bigger waste (KSYM_SYMBOL_LEN and TASK_COMM_LEN bytes respetively) and the breakpoint structure size is tiny in comparison.
The type information is an arch-specific field and it encapsulated within 'struct hw_breakpoint'. In any case this solution needs improvement more in terms of getting the tracer infrastructure to identify stale entries (belonging to removed symbols) and prevent their display. The existing trace logs contain huge redundant information due to multiple copies of the same data and it would yield much better results than such trivial optimisations. The ksym_tracer, I believe, has just unearthed the opportunity for the same as it is (is it?) the first such ftrace plugin to partially remove entries.
Thanks,
K.Prasad
Quoted 49 lines · K.Prasad
Quoted 41 lines · Frederic Weisbecker
Quoted 30 lines · K.Prasad
This patch fixes two issues reported with ksym tracer, seen after removal of a symbol from the tracer i) Concatenation of trace logs into a single line ii) Machine stall seen when logs are viewed using '
trace_pipe'.Known issue: Upon removal, 'cat
trace_pipe' (without any preceding command and any further output in the trace buffer) responds to SIGTERM quickly but only after an indeterminate amount of delay for SIGINT.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace.h | 12 +++++++++--- kernel/trace/trace_ksym.c | 11 ++++------- 2 files changed, 13 insertions(+), 10 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +1
-1@@ -71,7 +71,7 @@ void ksym_hbp_handler(struct hw_breakpoi {struct ring_buffer_event *event;struct trace_array *tr; -struct trace_ksym *entry; +struct trace_ksym_rb *entry;int pc;if (!ksym_tracing_enabled)kernel/trace/trace_ksym.c void ksym_hbp_handler(struct hw_breakpoi · line 90
Quoted 5 lines
kernel/trace/trace_ksym.c +1
-1@@ -87,7 +87,7 @@ void ksym_hbp_handler(struct hw_breakpoi entry =ring_buffer_event_data (event);strlcpy (entry->ksym_name, hbp->info.name,KSYM_SYMBOL_LEN ); - entry->ksym_hbp = hbp; +memcpy (&(entry->ksym_hbp), hbp,sizeof (struct hw_breakpoint ));That looks wasteful. You only need the type from the
arch_hw_breakpointand there you copy the whole generic breakpoint.Quoted 2 lines
Frederic.
Well, the entire copy operation for
ksym_nameandp_nameis indeed a much bigger waste (KSYM_SYMBOL_LENandTASK_COMM_LENbytes respetively) and the breakpoint structure size is tiny in comparison.
Quoted 9 lines
The type information is an arch-specific field and it encapsulated within 'struct
hw_breakpoint'. In any case this solution needs improvement more in terms of getting the tracer infrastructure to identify stale entries (belonging to removed symbols) and prevent their display. The existing trace logs contain huge redundant information due to multiple copies of the same data and it would yield much better results than such trivial optimisations. Theksym_tracer, I believe, has just unearthed the opportunity for the same as it is (is it?) the first such ftrace plugin to partially remove entries.
Well, we had such issue by the past and the problem can't
be easily solvable. Traces can possibily stay in the ring buffer for a while until they become consumed by a reader.
It's safe and keep the things simple to just copy the required informations in the ring buffer.
Also your type is indeed arch dependent, but copying struct hw_breakpoint instead of the hw_breakpoint in the ring buffer doesn't solve the problem, you'll still have some #ifdef ARCH once you'll need to support other archs.
Copying the whole struct breakpoint in the ring buffer doesn't make sense. You're even copying the address of the triggered callback whereas all you need is only the type field from the struct arch_hw_breakpoint ?
Btw, that makes me remind that I really think this hardware breakpoint API should provide a real abstraction of the arch dependent hardware breakpoints.
Quoted 4 lines
Thanks,
K.Prasad
PATCH 2/3 ksym_tracer: Allow bulk removal using empty or wildcard string input
In reply to a message not in the archiveAccept an empty string or wildcard input (of the form *:---) as input for ksym_trace_filter to perform bulk removal of all monitored entries.
--- kernel/trace/trace_ksym.c | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 33 deletions(-)
kernel/trace/trace_ksym.c +42 -33
@@ -138,6 +138,34 @@ static int ksym_trace_get_access_type(chreturn access; } +static void ksym_trace_reset (struct trace_array *tr) +{ +struct trace_ksym *entry; +struct hlist_node *node, *node1; + + ksym_tracing_enabled =0 ; + +mutex_lock (&ksym_tracer_mutex); +hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, + ksym_hlist) { +unregister_kernel_hw_breakpoint (entry->ksym_hbp); + ksym_filter_entry_count--; +hlist_del_rcu (&(entry->ksym_hlist)); +synchronize_rcu (); +/* Free the 'input_string' only if reset +* after startup self-test +*/ +#ifdef CONFIG_FTRACE_SELFTEST +if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , + strlen(KSYM_SELFTEST_ENTRY)) != 0) +#endif /* CONFIG_FTRACE_SELFTEST*/ +kfree (entry->ksym_hbp->info.name); +kfree (entry->ksym_hbp); +kfree (entry); + } +mutex_unlock (&ksym_tracer_mutex); +} +/* * There can be several possible malformed requests and we attempt to capture * all of them. We enumerate some of the rules
@@ -163,7 +191,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */ if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" )))) goto return_code; ret =ksym_trace_get_access_type (input_string);
@@ -211,6 +239,7 @@ int process_new_ksym_entry(char *ksymnam }hlist_add_head_rcu (&(entry->ksym_hlist), &ksym_filter_head); ksym_filter_entry_count++; + ksym_tracing_enabled =1 ;return 0 ; }
@@ -220,7 +249,7 @@ static ssize_t ksym_trace_filter_read(st {struct trace_ksym *entry;struct hlist_node *node; -char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ]; +char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ] ="\0" ;ssize_t ret, cnt =0 ;mutex_lock (&ksym_tracer_mutex);
@@ -251,9 +280,11 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if (count ==1 ) { +ksym_trace_reset (NULL ); +return count; + } input_string =kzalloc (count,GFP_KERNEL );if (!input_string)
@@ -269,6 +300,12 @@ static ssize_t ksym_trace_filter_write(skfree (input_string);return ret; } +/* Clear all breakpoints if echo "*:---" > ksym_trace_filter */ +if ((strncmp (ksymname,"*" ,strlen ("*" )) ==0 ) && (op ==0 )) { +ksym_trace_reset (NULL ); +kfree (input_string); +return count; + }mutex_lock (&ksym_tracer_mutex);
@@ -325,34 +362,6 @@ static const struct file_operations ksym .write = ksym_trace_filter_write, }; -static void ksym_trace_reset (struct trace_array *tr) -{ -struct trace_ksym *entry; -struct hlist_node *node, *node1; - - ksym_tracing_enabled =0 ; - -mutex_lock (&ksym_tracer_mutex); -hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, - ksym_hlist) { -unregister_kernel_hw_breakpoint (entry->ksym_hbp); - ksym_filter_entry_count--; -hlist_del_rcu (&(entry->ksym_hlist)); -synchronize_rcu (); -/* Free the 'input_string' only if reset -* after startup self-test -*/ -#ifdef CONFIG_FTRACE_SELFTEST -if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , - strlen(KSYM_SELFTEST_ENTRY)) != 0) -#endif /* CONFIG_FTRACE_SELFTEST*/ -kfree (entry->ksym_hbp->info.name); -kfree (entry->ksym_hbp); -kfree (entry); - } -mutex_unlock (&ksym_tracer_mutex); -} -static int ksym_trace_init (struct trace_array *tr) {int cpu, ret =0 ;
Re: [Patch 2/3] ksym_tracer: Allow bulk removal using empty or wildcard string input
Quoted 48 lines · K.Prasad
Accept an empty string or wildcard input (of the form *:---) as input for
ksym_trace_filterto perform bulk removal of all monitored entries.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 33 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +28
-0@@ -138,6 +138,34 @@ static int ksym_trace_get_access_type(chreturn access; } +static void ksym_trace_reset (struct trace_array *tr) +{ +struct trace_ksym *entry; +struct hlist_node *node, *node1; + + ksym_tracing_enabled =0 ; + +mutex_lock (&ksym_tracer_mutex); +hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, + ksym_hlist) { +unregister_kernel_hw_breakpoint (entry->ksym_hbp); + ksym_filter_entry_count--; +hlist_del_rcu (&(entry->ksym_hlist)); +synchronize_rcu (); +/* Free the 'input_string' only if reset +* after startup self-test +*/ +#ifdef CONFIG_FTRACE_SELFTEST +if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , + strlen(KSYM_SELFTEST_ENTRY)) != 0) +#endif /* CONFIG_FTRACE_SELFTEST*/ +kfree (entry->ksym_hbp->info.name); +kfree (entry->ksym_hbp); +kfree (entry); + } +mutex_unlock (&ksym_tracer_mutex); +} +/* * There can be several possible malformed requests and we attempt to capture * all of them. We enumerate some of the rules
kernel/trace/trace_ksym.c static int parse_ksym_trace_str(char *in · line 194
Quoted 5 lines
kernel/trace/trace_ksym.c +1
-1@@ -163,7 +191,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */ if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" ))))
I guess you can just use strcmp() here, no? As in other use of this same strncmp() in this patch.
Quoted 22 lines
kernel/trace/trace_ksym.c +2
-1goto return_code; ret =ksym_trace_get_access_type (input_string);@@ -211,6 +239,7 @@ int process_new_ksym_entry(char *ksymnam }hlist_add_head_rcu (&(entry->ksym_hlist), &ksym_filter_head); ksym_filter_entry_count++; + ksym_tracing_enabled =1 ;return 0 ; }@@ -220,7 +249,7 @@ static ssize_t ksym_trace_filter_read(st {struct trace_ksym *entry;struct hlist_node *node; -char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ]; +char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ] ="\0" ;ssize_t ret, cnt =0 ;mutex_lock (&ksym_tracer_mutex);@@ -251,9 +280,11 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;
kernel/trace/trace_ksym.c static ssize_t ksym_trace_filter_write(s · line 287
Quoted 10 lines
kernel/trace/trace_ksym.c +5
-3int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if (count ==1 ) { +ksym_trace_reset (NULL ); +return count; + }
And then if I do:
echo -n 1 > ksym_trace_filter
That would also clean these breakpoints?
Frederic.
Quoted 52 lines
kernel/trace/trace_ksym.c +6
-28input_string =kzalloc (count,GFP_KERNEL );if (!input_string)@@ -269,6 +300,12 @@ static ssize_t ksym_trace_filter_write(skfree (input_string);return ret; } +/* Clear all breakpoints if echo "*:---" > ksym_trace_filter */ +if ((strncmp (ksymname,"*" ,strlen ("*" )) ==0 ) && (op ==0 )) { +ksym_trace_reset (NULL ); +kfree (input_string); +return count; + }mutex_lock (&ksym_tracer_mutex);@@ -325,34 +362,6 @@ static const struct file_operations ksym .write = ksym_trace_filter_write, }; -static void ksym_trace_reset (struct trace_array *tr) -{ -struct trace_ksym *entry; -struct hlist_node *node, *node1; - - ksym_tracing_enabled =0 ; - -mutex_lock (&ksym_tracer_mutex); -hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, - ksym_hlist) { -unregister_kernel_hw_breakpoint (entry->ksym_hbp); - ksym_filter_entry_count--; -hlist_del_rcu (&(entry->ksym_hlist)); -synchronize_rcu (); -/* Free the 'input_string' only if reset -* after startup self-test -*/ -#ifdef CONFIG_FTRACE_SELFTEST -if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , - strlen(KSYM_SELFTEST_ENTRY)) != 0) -#endif /* CONFIG_FTRACE_SELFTEST*/ -kfree (entry->ksym_hbp->info.name); -kfree (entry->ksym_hbp); -kfree (entry); - } -mutex_unlock (&ksym_tracer_mutex); -} -static int ksym_trace_init (struct trace_array *tr) {int cpu, ret =0 ;
Re: [Patch 2/3] ksym_tracer: Allow bulk removal using empty or wildcard string input
Quoted 9 lines · Frederic Weisbecker
Quoted 8 lines · K.Prasad
Accept an empty string or wildcard input (of the form *:---) as input for
ksym_trace_filterto perform bulk removal of all monitored entries.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 33 deletions(-)
<snipped>
Quoted 5 lines
Quoted 5 lines
/*
* There can be several possible malformed requests and we attempt to capture
* all of them. We enumerate some of the ruleskernel/trace/trace_ksym.c +0
-0@@ -163,7 +191,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */
Quoted 9 lines
kernel/trace/trace_ksym.c static int parse_ksym_trace_str(char *in · line 194
Quoted 4 lines
kernel/trace/trace_ksym.c +1
-1if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" ))))I guess you can just use
strcmp()here, no? As in other use of this samestrncmp()in this patch.
I see both functions being used in the kernel at random (what happened to all the buffer-overflow scare?)! Isn't an strncmp() preferred over the plain strcmp() (atleast it cannot be the other way round)?
Quoted 16 lines
kernel/trace/trace_ksym.c static ssize_t ksym_trace_filter_write(s · line 287
Quoted 12 lines
kernel/trace/trace_ksym.c +5
-3@@ -251,9 +280,11 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if (count ==1 ) { +ksym_trace_reset (NULL ); +return count; + }And then if I do:
Quoted 6 lines
echo -n 1 >
ksym_trace_filterThat would also clean these breakpoints?
Frederic.
No, I did not think about this scenario and I am changing the patch to handle such a case. Also, the value of 'count' cannot be '0' as such inputs are ignored (e.g. echo -n "" > ksym_trace_filter is simply ignored) and so I'm removing the check for the same.
Please find a new version of the patch sent here:
http://lkml.org/lkml/2009/6/20/73.
Thanks,
K.Prasad
Re: [Patch 2/3] ksym_tracer: Allow bulk removal using empty or wildcard string input
Quoted 26 lines · K.Prasad
Quoted 9 lines · Frederic Weisbecker
Quoted 8 lines · K.Prasad
Accept an empty string or wildcard input (of the form *:---) as input for
ksym_trace_filterto perform bulk removal of all monitored entries.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 33 deletions(-)<snipped>
Quoted 5 lines
Quoted 5 lines
/*
* There can be several possible malformed requests and we attempt to capture
* all of them. We enumerate some of the ruleskernel/trace/trace_ksym.c +0
-0@@ -163,7 +191,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */ Quoted 9 lines
kernel/trace/trace_ksym.c static int parse_ksym_trace_str(char *in · line 194
Quoted 4 lines
kernel/trace/trace_ksym.c +1
-1if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" ))))I guess you can just use
strcmp()here, no? As in other use of this samestrncmp()in this patch.
Quoted 3 lines
I see both functions being used in the kernel at random (what happened to all the buffer-overflow scare?)! Isn't an
strncmp()preferred over the plainstrcmp()(atleast it cannot be the other way round)?
I guess it depends if the string has complicated origins :) But a plain 1 for the len would be sufficient.
Or here you can just check **ksymname != '*'
Anyway, I'm arguing about neat things that don't matter that much.
Thanks.
Quoted 35 lines
Quoted 16 lines
kernel/trace/trace_ksym.c static ssize_t ksym_trace_filter_write(s · line 287
Quoted 12 lines
kernel/trace/trace_ksym.c +5
-3@@ -251,9 +280,11 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if (count ==1 ) { +ksym_trace_reset (NULL ); +return count; + }And then if I do:
Quoted 6 lines
echo -n 1 >
ksym_trace_filterThat would also clean these breakpoints?
Frederic.
No, I did not think about this scenario and I am changing the patch to handle such a case. Also, the value of 'count' cannot be '0' as such inputs are ignored (e.g. echo -n "" >
ksym_trace_filteris simply ignored) and so I'm removing the check for the same.Please find a new version of the patch sent here:
http://lkml.org/lkml/2009/6/20/73.Thanks,
K.Prasad
PATCH 2/3 ksym_tracer: Allow bulk removal using empty or wildcard string input - ver II
In reply to K.PrasadAccept an empty string or wildcard input (of the form *:---) as input for ksym_trace_filter to perform bulk removal of all monitored entries.
--- kernel/trace/trace_ksym.c | 77 +++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 34 deletions(-)
kernel/trace/trace_ksym.c +43 -34
@@ -23,6 +23,7 @@#include <linux/debugfs.h> #include <linux/ftrace.h> #include <linux/module.h> +#include <linux/ctype.h> #include <linux/fs.h> #include "trace_output.h"
@@ -138,6 +139,34 @@ static int ksym_trace_get_access_type(chreturn access; } +static void ksym_trace_reset (struct trace_array *tr) +{ +struct trace_ksym *entry; +struct hlist_node *node, *node1; + + ksym_tracing_enabled =0 ; + +mutex_lock (&ksym_tracer_mutex); +hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, + ksym_hlist) { +unregister_kernel_hw_breakpoint (entry->ksym_hbp); + ksym_filter_entry_count--; +hlist_del_rcu (&(entry->ksym_hlist)); +synchronize_rcu (); +/* Free the 'input_string' only if reset +* after startup self-test +*/ +#ifdef CONFIG_FTRACE_SELFTEST +if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , + strlen(KSYM_SELFTEST_ENTRY)) != 0) +#endif /* CONFIG_FTRACE_SELFTEST*/ +kfree (entry->ksym_hbp->info.name); +kfree (entry->ksym_hbp); +kfree (entry); + } +mutex_unlock (&ksym_tracer_mutex); +} +/* * There can be several possible malformed requests and we attempt to capture * all of them. We enumerate some of the rules
@@ -163,7 +192,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */ if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" )))) goto return_code; ret =ksym_trace_get_access_type (input_string);
@@ -211,6 +240,7 @@ int process_new_ksym_entry(char *ksymnam }hlist_add_head_rcu (&(entry->ksym_hlist), &ksym_filter_head); ksym_filter_entry_count++; + ksym_tracing_enabled =1 ;return 0 ; }
@@ -220,7 +250,7 @@ static ssize_t ksym_trace_filter_read(st {struct trace_ksym *entry;struct hlist_node *node; -char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ]; +char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ] ="\0" ;ssize_t ret, cnt =0 ;mutex_lock (&ksym_tracer_mutex);
@@ -251,10 +281,6 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; - input_string =kzalloc (count,GFP_KERNEL );if (!input_string)return -ENOMEM ;
@@ -263,12 +289,23 @@ static ssize_t ksym_trace_filter_write(skfree (input_string);return -EFAULT ; } +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if ((count ==1 ) && !isalnum (input_string[0 ])) { +ksym_trace_reset (NULL ); +return count; + } ret = op =parse_ksym_trace_str (input_string, &ksymname, &ksym_addr);if (ret <0 ) {kfree (input_string);return ret; } +/* Clear all breakpoints if echo "*:---" > ksym_trace_filter */ +if ((strncmp (ksymname,"*" ,strlen ("*" )) ==0 ) && (op ==0 )) { +ksym_trace_reset (NULL ); +kfree (input_string); +return count; + }mutex_lock (&ksym_tracer_mutex);
@@ -325,34 +362,6 @@ static const struct file_operations ksym .write = ksym_trace_filter_write, }; -static void ksym_trace_reset (struct trace_array *tr) -{ -struct trace_ksym *entry; -struct hlist_node *node, *node1; - - ksym_tracing_enabled =0 ; - -mutex_lock (&ksym_tracer_mutex); -hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, - ksym_hlist) { -unregister_kernel_hw_breakpoint (entry->ksym_hbp); - ksym_filter_entry_count--; -hlist_del_rcu (&(entry->ksym_hlist)); -synchronize_rcu (); -/* Free the 'input_string' only if reset -* after startup self-test -*/ -#ifdef CONFIG_FTRACE_SELFTEST -if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , - strlen(KSYM_SELFTEST_ENTRY)) != 0) -#endif /* CONFIG_FTRACE_SELFTEST*/ -kfree (entry->ksym_hbp->info.name); -kfree (entry->ksym_hbp); -kfree (entry); - } -mutex_unlock (&ksym_tracer_mutex); -} -static int ksym_trace_init (struct trace_array *tr) {int cpu, ret =0 ;
Re: [Patch 2/3] ksym_tracer: Allow bulk removal using empty or wildcard string input - ver II
Quoted 99 lines · K.Prasad
Accept an empty string or wildcard input (of the form *:---) as input for
ksym_trace_filterto perform bulk removal of all monitored entries.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 77 +++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 34 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +35
-6@@ -23,6 +23,7 @@#include <linux/debugfs.h> #include <linux/ftrace.h> #include <linux/module.h> +#include <linux/ctype.h> #include <linux/fs.h> #include "trace_output.h" @@ -138,6 +139,34 @@ static int ksym_trace_get_access_type(chreturn access; } +static void ksym_trace_reset (struct trace_array *tr) +{ +struct trace_ksym *entry; +struct hlist_node *node, *node1; + + ksym_tracing_enabled =0 ; + +mutex_lock (&ksym_tracer_mutex); +hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, + ksym_hlist) { +unregister_kernel_hw_breakpoint (entry->ksym_hbp); + ksym_filter_entry_count--; +hlist_del_rcu (&(entry->ksym_hlist)); +synchronize_rcu (); +/* Free the 'input_string' only if reset +* after startup self-test +*/ +#ifdef CONFIG_FTRACE_SELFTEST +if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , + strlen(KSYM_SELFTEST_ENTRY)) != 0) +#endif /* CONFIG_FTRACE_SELFTEST*/ +kfree (entry->ksym_hbp->info.name); +kfree (entry->ksym_hbp); +kfree (entry); + } +mutex_unlock (&ksym_tracer_mutex); +} +/* * There can be several possible malformed requests and we attempt to capture * all of them. We enumerate some of the rules @@ -163,7 +192,7 @@ static int parse_ksym_trace_str(char *in/* Check for malformed request: (2), (1) and (5) */ if ((!input_string) || (strlen (input_string) != (KSYM_TRACER_OP_LEN +1 )) || - (*addr ==0 )) + ((*addr ==0 ) &&strncmp (*ksymname,"*" ,strlen ("*" )))) goto return_code; ret =ksym_trace_get_access_type (input_string);@@ -211,6 +240,7 @@ int process_new_ksym_entry(char *ksymnam }hlist_add_head_rcu (&(entry->ksym_hlist), &ksym_filter_head); ksym_filter_entry_count++; + ksym_tracing_enabled =1 ;return 0 ; }@@ -220,7 +250,7 @@ static ssize_t ksym_trace_filter_read(st {struct trace_ksym *entry;struct hlist_node *node; -char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ]; +char buf[KSYM_FILTER_ENTRY_LEN *KSYM_TRACER_MAX ] ="\0" ;ssize_t ret, cnt =0 ;mutex_lock (&ksym_tracer_mutex);@@ -251,10 +281,6 @@ static ssize_t ksym_trace_filter_write(sunsigned long ksym_addr =0 ;int ret, op, changed =0 ; -/* Ignore echo "" > ksym_trace_filter */ -if (count ==0 ) -return 0 ; - input_string =kzalloc (count,GFP_KERNEL );if (!input_string)return -ENOMEM ;@@ -263,12 +289,23 @@ static ssize_t ksym_trace_filter_write(skfree (input_string);return -EFAULT ; } +/* Clear all breakpoint requests if echo "" > ksym_trace_filter */ +if ((count ==1 ) && !isalnum (input_string[0 ])) { +ksym_trace_reset (NULL );
kernel/trace/trace_ksym.c static ssize_t ksym_trace_filter_write(s · line 304
Quoted 10 lines
kernel/trace/trace_ksym.c +4
-0+return count; + } ret = op =parse_ksym_trace_str (input_string, &ksymname, &ksym_addr);if (ret <0 ) {kfree (input_string);return ret; } +/* Clear all breakpoints if echo "*:---" > ksym_trace_filter */ +if ((strncmp (ksymname,"*" ,strlen ("*" )) ==0 ) && (op ==0 )) {
If you returned -EINVAL above,
if (*ksymname == '*' && !op)
..would be shorter
Also that doesn't check if symname = "*something:---"
Quoted 42 lines
kernel/trace/trace_ksym.c +4
-28+ksym_trace_reset (NULL ); +kfree (input_string); +return count; + }mutex_lock (&ksym_tracer_mutex);@@ -325,34 +362,6 @@ static const struct file_operations ksym .write = ksym_trace_filter_write, }; -static void ksym_trace_reset (struct trace_array *tr) -{ -struct trace_ksym *entry; -struct hlist_node *node, *node1; - - ksym_tracing_enabled =0 ; - -mutex_lock (&ksym_tracer_mutex); -hlist_for_each_entry_safe (entry, node, node1, &ksym_filter_head, - ksym_hlist) { -unregister_kernel_hw_breakpoint (entry->ksym_hbp); - ksym_filter_entry_count--; -hlist_del_rcu (&(entry->ksym_hlist)); -synchronize_rcu (); -/* Free the 'input_string' only if reset -* after startup self-test -*/ -#ifdef CONFIG_FTRACE_SELFTEST -if (strncmp (entry->ksym_hbp->info.name,KSYM_SELFTEST_ENTRY , - strlen(KSYM_SELFTEST_ENTRY)) != 0) -#endif /* CONFIG_FTRACE_SELFTEST*/ -kfree (entry->ksym_hbp->info.name); -kfree (entry->ksym_hbp); -kfree (entry); - } -mutex_unlock (&ksym_tracer_mutex); -} -static int ksym_trace_init (struct trace_array *tr) {int cpu, ret =0 ;
PATCH 3/3 ksym_tracer: Documentation containing usage guide for ksym tracer
In reply to a message not in the archiveThis patch adds documentation for the ksym tracer plugin in ftrace. It contains a minimal usage guide for the same.
--- Documentation/trace/ksym_tracer.txt | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+)
Documentation/trace/ksym_tracer.txt new file +90 -0
@@ -0,0 +1,90 @@ + ksym_tracer - Kernel Symbol Tracer + ---------------------------------- + K.Prasad <[email protected]> +I. Introduction +=============== + +ksym_tracer uses the Hardware Breakpoint interface in the kernel to monitor +kernel variables for memory access operations such as read and write. + +The number of kernel variables that can be monitored simultaneously is directly +dependant upon the available free debug registers on the processor at the time +of request. + +If the memory access operation on the variable added to the ksym tracer occurs, +a trace containing details such as symbol name, function that accessed the +variable, PID and cpu are logged onto the ring buffer. + +ksym tracer's use-cases can include the following: +- Debug memory corruption issues: Trace write operations on a variable that is +known to get corrupted on a system under debug +- Profile variables: Understand how often a variable is being used by the +system (such as read-mostly or write-mostly). + +II. Usage Guide +=============== + +The following is a list of steps to monitor a kernel variable using ksym +tracer. It is illustrated by taking 'pid_max' as the kernel variable to be +monitored. + +1) Mount debugfs to a directory on your system, say /sys/kernel/debug +2) The memory access operations for which you want to monitor the variable can +be specified as follows: + <ksym_name>:rwx +The type of operation can be specified in three characters, and the valid +combinations are dependant on the host processor. On x86 processors, the valid +requests are '-w-' (write operations only) and 'rw-' (read or write +operations). So to monitor 'pid_max' for write the corresponding string would be +"pid_max:-w-". +3) Do 'cat available_tracers' in <debugfs_mount>/tracing/ to check if +'ksym_tracer' is listed as one of the available tracer. A 'ksym_trace_filter' +file should be available to accept the kernel symbol inputs. +4) echo the kernel symbol along with the type operation string to +ksym_trace_filter. +e.g. echo pid_max:-w- > ksym_trace_filter +Lookout for any error messages in the shell after the echo and in dmesg that +indicates a failure of the request. +5) Make sure that tracing is enabled by checking for 1 in 'tracing_enabled' +file, if not 'echo 1 > tracing_enabled' to start tracing. + +Now a breakpoint over the kernel symbol (pid_max in this example) is active and +is monitoring for any operations of the requested type. Upon such an operation, +a trace containing the following information is logged and is available in the +'trace' file. + +For instance, in case of the above example, the 'trace' file looks as shown +below: +# echo 32621 > /proc/sys/kernel/pid_max +# cat trace +# tracer: ksym_tracer +# +# TASK-PID CPU# Symbol Type Function +# | | | | | +bash 3027 2 pid_max W do_proc_dointvec_minmax_conv +# + +In order to remove the trace, the kernel symbol should be echoed with all +operators set to '-' like this '<ksym_name>:---'. For example, +echo pid_max:--- > ksym_trace_filter. + +Alternatively if you choose to unregister all requests simultaneously you may +do either of the following; +echo > ksym_trace_filter +echo *:--- > ksym_trace_filter + +The ksym tracer also provides a consolidated hit-counter that indicates the +number of times a variable was subjected to the specified memory access +operation. This is available in the file +<debugfs_mount>/tracing/trace_stat/ksym_tracer. + +Limitations +------------ +The known limitations of the ksym tracer are listed below. Efforts are +through to support many of feature limitations. + +- ksym tracer can monitor only primary data types (such as int, char) and not +arrays or instances or members of any structure. +- Symbols that are defined in modules are not resolved by ksym tracer's symbol +resolution mechanism. +- Monitoring either read or write only is not supported on all processors.
Re: [Patch 3/3] ksym_tracer: Documentation containing usage guide for ksym tracer
Quoted 94 lines · K.Prasad
This patch adds documentation for the ksym tracer plugin in ftrace. It contains a minimal usage guide for the same.
Signed-off-by K.Prasad <[email protected]>--- Documentation/trace/ksym_tracer.txt | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+)Index:
linux-2.6-tip.hbkpt/Documentation/trace/ksym_tracer.txt
===================================================================Documentation/trace/ksym_tracer.txt new file +80
-0@@ -0,0 +1,90 @@ + ksym_tracer - Kernel Symbol Tracer + ---------------------------------- + K.Prasad <[email protected]> +I. Introduction +=============== + +ksym_tracer uses the Hardware Breakpoint interface in the kernel to monitor +kernel variables for memory access operations such as read and write. + +The number of kernel variables that can be monitored simultaneously is directly +dependant upon the available free debug registers on the processor at the time +of request. + +If the memory access operation on the variable added to the ksym tracer occurs, +a trace containing details such as symbol name, function that accessed the +variable, PID and cpu are logged onto the ring buffer. + +ksym tracer's use-cases can include the following: +- Debug memory corruption issues: Trace write operations on a variable that is +known to get corrupted on a system under debug +- Profile variables: Understand how often a variable is being used by the +system (such as read-mostly or write-mostly). + +II. Usage Guide +=============== + +The following is a list of steps to monitor a kernel variable using ksym +tracer. It is illustrated by taking 'pid_max' as the kernel variable to be +monitored. + +1) Mount debugfs to a directory on your system, say /sys/kernel/debug +2) The memory access operations for which you want to monitor the variable can +be specified as follows: + <ksym_name>:rwx +The type of operation can be specified in three characters, and the valid +combinations are dependant on the host processor. On x86 processors, the valid +requests are '-w-' (write operations only) and 'rw-' (read or write +operations). So to monitor 'pid_max' for write the corresponding string would be +"pid_max:-w-". +3) Do 'cat available_tracers' in <debugfs_mount>/tracing/ to check if +'ksym_tracer' is listed as one of the available tracer. A 'ksym_trace_filter' +file should be available to accept the kernel symbol inputs. +4) echo the kernel symbol along with the type operation string to +ksym_trace_filter. +e.g. echo pid_max:-w- > ksym_trace_filter +Lookout for any error messages in the shell after the echo and in dmesg that +indicates a failure of the request. +5) Make sure that tracing is enabled by checking for 1 in 'tracing_enabled' +file, if not 'echo 1 > tracing_enabled' to start tracing. + +Now a breakpoint over the kernel symbol (pid_max in this example) is active and +is monitoring for any operations of the requested type. Upon such an operation, +a trace containing the following information is logged and is available in the +'trace' file. + +For instance, in case of the above example, the 'trace' file looks as shown +below: +# echo 32621 > /proc/sys/kernel/pid_max +# cat trace +# tracer: ksym_tracer +# +# TASK-PID CPU# Symbol Type Function +# | | | | | +bash 3027 2 pid_max W do_proc_dointvec_minmax_conv +# + +In order to remove the trace, the kernel symbol should be echoed with all +operators set to '-' like this '<ksym_name>:---'. For example, +echo pid_max:--- > ksym_trace_filter. + +Alternatively if you choose to unregister all requests simultaneously you may +do either of the following; +echo > ksym_trace_filter +echo *:--- > ksym_trace_filter + +The ksym tracer also provides a consolidated hit-counter that indicates the +number of times a variable was subjected to the specified memory access +operation. This is available in the file +<debugfs_mount>/tracing/trace_stat/ksym_tracer. +
Documentation/trace/ksym_tracer.txt line 90
Quoted 10 lines
Documentation/trace/ksym_tracer.txt new file +10
-0+Limitations +------------ +The known limitations of the ksym tracer are listed below. Efforts are +through to support many of feature limitations. + +- ksym tracer can monitor only primary data types (such as int, char) and not +arrays or instances or members of any structure. +- Symbols that are defined in modules are not resolved by ksym tracer's symbol +resolution mechanism. +- Monitoring either read or write only is not supported on all processors.
Nice documentation, thanks!