ksym_tracer:Fix line-wrapping after removal of ksym tracer entry
A message that is not in the archive (sent off-list or not yet received): 20090616225257.041883212@prasadkr_t60p.in.ibm.com
PATCH 1/2 ksym_tracer:Fix line-wrapping after removal of ksym tracer entry
Removal of a ksym entry results in missing information and an early return with TRACE_TYPE_PARTIAL_LINE code (minus the line return). This patch modifies the output function to unconditionally add a line return irrespective of the return code.
--- kernel/trace/trace_ksym.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)
kernel/trace/trace_ksym.c +15 -12
@@ -391,29 +391,32 @@ static enum print_line_t ksym_trace_outp ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; +if (!ret) { + ret =TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; + } + ret =TRACE_TYPE_PARTIAL_LINE ;switch (field->ksym_hbp->info.type) {case HW_BREAKPOINT_WRITE : - ret =trace_seq_printf (s," W " ); +if (!trace_seq_printf (s," W " )) + goto err_ret;break ;case HW_BREAKPOINT_RW : - ret =trace_seq_printf (s," RW " ); +if (!trace_seq_printf (s," RW " )) + goto err_ret;break ;default : -return TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; } -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; -sprint_symbol (str, field->ip); ret =trace_seq_printf (s,"%-20s\n" , str); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; - -return TRACE_TYPE_HANDLED ; +if (ret) +return TRACE_TYPE_HANDLED ; +err_ret: +trace_seq_printf (s,"\n" ); +return ret; } struct tracer ksym_tracer __read_mostly =
Quoted 46 lines · K.Prasad
Removal of a ksym entry results in missing information and an early return with
TRACE_TYPE_PARTIAL_LINEcode (minus the line return). This patch modifies the output function to unconditionally add a line return irrespective of the return code.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +10
-8@@ -391,29 +391,32 @@ static enum print_line_t ksym_trace_outp ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; +if (!ret) { + ret =TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; + } + ret =TRACE_TYPE_PARTIAL_LINE ;switch (field->ksym_hbp->info.type) {case HW_BREAKPOINT_WRITE : - ret =trace_seq_printf (s," W " ); +if (!trace_seq_printf (s," W " )) + goto err_ret;break ;case HW_BREAKPOINT_RW : - ret =trace_seq_printf (s," RW " ); +if (!trace_seq_printf (s," RW " )) + goto err_ret;break ;default : -return TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; } -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; -sprint_symbol (str, field->ip);
kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 419
Quoted 10 lines
kernel/trace/trace_ksym.c +5
-4ret =trace_seq_printf (s,"%-20s\n" , str); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; - -return TRACE_TYPE_HANDLED ; +if (ret) +return TRACE_TYPE_HANDLED ; +err_ret: +trace_seq_printf (s,"\n" ); +return ret;
As told in my email just before, TRACE_TYPE_PARTIAL_LINE won't print a truncated line or a partial line. Instead, it will be ignored and entirely retried later.
Then your newline will be ignored.
..unless we have a bug in trace.c
I'm not sure what it the origin of the concatenated printed entries Ingo has reported.
Did you reproduce it and then this patch fixed it?
Quoted 4 lines
kernel/trace/trace_ksym.c +0
-0} struct tracer ksym_tracer __read_mostly =
Quoted 64 lines · Frederic Weisbecker
Quoted 46 lines · K.Prasad
Removal of a ksym entry results in missing information and an early return with
TRACE_TYPE_PARTIAL_LINEcode (minus the line return). This patch modifies the output function to unconditionally add a line return irrespective of the return code.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +10
-8@@ -391,29 +391,32 @@ static enum print_line_t ksym_trace_outp ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; +if (!ret) { + ret =TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; + } + ret =TRACE_TYPE_PARTIAL_LINE ;switch (field->ksym_hbp->info.type) {case HW_BREAKPOINT_WRITE : - ret =trace_seq_printf (s," W " ); +if (!trace_seq_printf (s," W " )) + goto err_ret;break ;case HW_BREAKPOINT_RW : - ret =trace_seq_printf (s," RW " ); +if (!trace_seq_printf (s," RW " )) + goto err_ret;break ;default : -return TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; } -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; -sprint_symbol (str, field->ip);kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 419
Quoted 10 lines
kernel/trace/trace_ksym.c +5
-4ret =trace_seq_printf (s,"%-20s\n" , str); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; - -return TRACE_TYPE_HANDLED ; +if (ret) +return TRACE_TYPE_HANDLED ; +err_ret: +trace_seq_printf (s,"\n" ); +return ret;As told in my email just before,
TRACE_TYPE_PARTIAL_LINEwon't print a truncated line or a partial line. Instead, it will be ignored and entirely retried later.
Then your newline will be ignored.
Quoted 2 lines
..unless we have a bug in
trace.c
Yes, I see that TRACE_TYPE_PARTIAL_LINE isn't the cause but the early return that happens (without printing the \n character) because of missing data.
Quoted 3 lines
I'm not sure what it the origin of the concatenated printed entries Ingo has reported.
The existing code in -tip reads like this:
397 switch (field->ksym_hbp->info.type) {
398 case HW_BREAKPOINT_WRITE:
399 ret = trace_seq_printf(s, " W ");
400 break;
401 case HW_BREAKPOINT_RW:
402 ret = trace_seq_printf(s, " RW ");
403 break;
404 default:
405 return TRACE_TYPE_PARTIAL_LINE;
406 }
407
408 if (!ret)
409 return TRACE_TYPE_PARTIAL_LINE;
410
411 sprint_symbol(str, field->ip); 412 ret = trace_seq_printf(s, "%-20s\n", str); 413 if (!ret)
414 return TRACE_TYPE_PARTIAL_LINE;
415
Si, if the control returns early at line 405 it skips the line at 412 which prints a newline character. Line 405 is executed if field->ksym_hbp->info.type is 0 which is true for an entry that is removed from the filter (e.g. echo jiffies:--- > ksym_trace_filter) and no amount of retries will help it get some new data (as it is already free-ed).
Quoted 2 lines
Did you reproduce it and then this patch fixed it?
Yes, it was reproduced and tested but actually it is the second patch that helps resolve this. The "trace_seq_printf(s, "\n");" isn't required at all.
I will re-send a new patch to you which fixes the line concatenation and machine stall issue (after symbol removal in ksym_tracer).
Thanks,
K.Prasad
Quoted 98 lines · K.Prasad
Quoted 64 lines · Frederic Weisbecker
Quoted 46 lines · K.Prasad
Removal of a ksym entry results in missing information and an early return with
TRACE_TYPE_PARTIAL_LINEcode (minus the line return). This patch modifies the output function to unconditionally add a line return irrespective of the return code.Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c +10
-8@@ -391,29 +391,32 @@ static enum print_line_t ksym_trace_outp ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; +if (!ret) { + ret =TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; + } + ret =TRACE_TYPE_PARTIAL_LINE ;switch (field->ksym_hbp->info.type) {case HW_BREAKPOINT_WRITE : - ret =trace_seq_printf (s," W " ); +if (!trace_seq_printf (s," W " )) + goto err_ret;break ;case HW_BREAKPOINT_RW : - ret =trace_seq_printf (s," RW " ); +if (!trace_seq_printf (s," RW " )) + goto err_ret;break ;default : -return TRACE_TYPE_PARTIAL_LINE ; + goto err_ret; } -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; -sprint_symbol (str, field->ip);kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 419
Quoted 10 lines
kernel/trace/trace_ksym.c +5
-4ret =trace_seq_printf (s,"%-20s\n" , str); -if (!ret) -return TRACE_TYPE_PARTIAL_LINE ; - -return TRACE_TYPE_HANDLED ; +if (ret) +return TRACE_TYPE_HANDLED ; +err_ret: +trace_seq_printf (s,"\n" ); +return ret;As told in my email just before,
TRACE_TYPE_PARTIAL_LINEwon't print a truncated line or a partial line. Instead, it will be ignored and entirely retried later.
Then your newline will be ignored.Quoted 2 lines
..unless we have a bug in
trace.cYes, I see that
TRACE_TYPE_PARTIAL_LINEisn't the cause but the early return that happens (without printing the \n character) because of missing data.Quoted 3 lines
I'm not sure what it the origin of the concatenated printed entries Ingo has reported.
The existing code in -tip reads like this:
397 switch (
field->ksym_hbp->info.type) {
398 caseHW_BREAKPOINT_WRITE:
399 ret =trace_seq_printf(s, " W ");
400 break;
401 caseHW_BREAKPOINT_RW:
402 ret =trace_seq_printf(s, " RW ");
403 break;
404 default:
405 returnTRACE_TYPE_PARTIAL_LINE;
406 }
407
408 if (!ret)
409 returnTRACE_TYPE_PARTIAL_LINE;
410411 sprint_symbol(str, field->ip); 412 ret = trace_seq_printf(s, "%-20s\n", str); 413 if (!ret)414 return
TRACE_TYPE_PARTIAL_LINE;
415
Quoted 6 lines
Si, if the control returns early at line 405 it skips the line at 412 which prints a newline character. Line 405 is executed if
field->ksym_hbp->info.typeis 0 which is true for an entry that is removed from the filter (e.g. echo jiffies:--- >ksym_trace_filter) and no amount of retries will help it get some new data (as it is already free-ed).
Ok, I guess a straightforward copy to the ring buffer (and not a pointer to your info) would solve it.
Quoted 9 lines
Quoted 2 lines
Did you reproduce it and then this patch fixed it?
Yes, it was reproduced and tested but actually it is the second patch that helps resolve this. The "
trace_seq_printf(s, "\n");" isn't required at all.I will re-send a new patch to you which fixes the line concatenation and machine stall issue (after symbol removal in
ksym_tracer).
Ok.
Thanks.
Quoted 3 lines
Thanks,
K.Prasad
PATCH 2/2 ksym_tracer:Handle machine stall when cat trace_pipe for ksym tracer
In reply to a message not in the archive'trace_pipe' does not handle a TRACE_TYPE_PARTIAL_LINE well and causes the machine to stall. While a TRACE_TYPE_UNHANDLED return causes the tracer to output unrelated data, a TRACE_TYPE_HANDLED return presents a clean output (minus all partial traces).
--- kernel/trace/trace_ksym.c | 6 ++++++ 1 file changed, 6 insertions(+)
kernel/trace/trace_ksym.c +6 -0
@@ -389,6 +389,12 @@ static enum print_line_t ksym_trace_outptrace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ; ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name);if (!ret) {
Re: [Patch 2/2] ksym_tracer:Handle machine stall when cat trace_pipe for ksym tracer
Quoted 2 lines · K.Prasad
'
trace_pipe' does not handle aTRACE_TYPE_PARTIAL_LINEwell and causes the machine to stall.
No, if it stalls here, it means it handles it well :)
Quoted 14 lines
While a
TRACE_TYPE_UNHANDLEDreturn causes the tracer to output unrelated data, aTRACE_TYPE_HANDLEDreturn presents a clean output (minus all partial traces).Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 6 ++++++ 1 file changed, 6 insertions(+)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================
kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 9 lines
kernel/trace/trace_ksym.c +6
-0@@ -389,6 +389,12 @@ static enum print_line_t ksym_trace_outptrace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;
Yeah that seems a good fix. But such silent ignored traces may hide bugs (current or futures).
Is it a common situation to have a zero ip or an undefined breakpoint type? How can that happen?
Thanks.
Quoted 4 lines
kernel/trace/trace_ksym.c +0
-0ret =trace_seq_printf (s,"%-15s %-5d %-3d %-20s " , field->p_name, entry->pid, iter->cpu, field->ksym_name);if (!ret) {
Re: [Patch 2/2] ksym_tracer:Handle machine stall when cat trace_pipe for ksym tracer
Quoted 8 lines · Frederic Weisbecker
Quoted 2 lines · K.Prasad
'
trace_pipe' does not handle aTRACE_TYPE_PARTIAL_LINEwell and causes the machine to stall.No, if it stalls here, it means it handles it well :)
Okay. As stated earlier, the cause is an incorrect TRACE_TYPE_PARTIAL_LINE return.
Quoted 31 lines
Quoted 14 lines
While a
TRACE_TYPE_UNHANDLEDreturn causes the tracer to output unrelated data, aTRACE_TYPE_HANDLEDreturn presents a clean output (minus all partial traces).Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 6 ++++++ 1 file changed, 6 insertions(+)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 9 lines
kernel/trace/trace_ksym.c +6
-0@@ -389,6 +389,12 @@ static enum print_line_t ksym_trace_outptrace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;Yeah that seems a good fix. But such silent ignored traces may hide bugs (current or futures).
Is it a common situation to have a zero ip or an undefined breakpoint type? How can that happen?
Quoted 2 lines
Thanks.
When a symbol is removed from ksym's trace list, it causes unregistration of the breakpoints and the breakpoint structure is kfree()-ed.
Now, looking at the relevant code in ksym_hbp_handler() in trace_ksym.c
88 entry = ring_buffer_event_data(event); 89 strlcpy(entry->ksym_name, hbp->info.name, KSYM_SYMBOL_LEN); 90 entry->ksym_hbp = hbp; 91 entry->ip = instruction_pointer(regs); 92 strlcpy(entry->p_name, current->comm, TASK_COMM_LEN);
While the ksym_name and p_name fields are copied into the ring buffer, only a pointer to the structure is provided (in line 90). So, when ksym_hbp is unregistered the fields contained in it, namely 'type' and 'ip' are lost (I'm not sure how the fields beyond 'field->ksym_hbp' is accessible in ksym_trace_output() when it is supposed to be already free).
This patch would cause an early return when such is the case. I also plan to remove the 'default case' in ksym_trace_output() as it was meant to handle a zero value for type (other values are eliminated at the time of breakpoint registration itself).
Please find a patch below that eliminates the two issues of output concatenation and machine stall, excepting for one issue which I'm unable to reason out.
A 'cat trace_pipe' done immediately after removal of an entry (without a preceding 'cat trace') terminates only with a SIGTERM (kill <pid>) and not a SIGINT (Ctrl+C). However, if the trace buffer receives any new data subsequently, it responds to the Ctrl+C signal. I would be glad to receive your suggestions in this regard.
===================================================================
kernel/trace/trace_ksym.c +6 -0
@@ -388,6 +388,12 @@ static enum print_line_t ksym_trace_outpreturn TRACE_TYPE_UNHANDLED ;trace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ; ret = trace_seq_printf(s, "%-15s %-5d %-3d %-20s ",
field->p_name,
entry->pid, iter->cpu,field->ksym_name);
@@ -401,10 +407,7 @@ static enum print_line_t ksym_trace_outp
case HW_BREAKPOINT_RW:
ret = trace_seq_printf(s, " RW "); break;
- default:
- return TRACE_TYPE_PARTIAL_LINE;
}
-
if (!ret)
return TRACE_TYPE_PARTIAL_LINE;
Thanks,
K.Prasad
Re: [Patch 2/2] ksym_tracer:Handle machine stall when cat trace_pipe for ksym tracer
Quoted 59 lines · K.Prasad
Quoted 8 lines · Frederic Weisbecker
Quoted 2 lines · K.Prasad
'
trace_pipe' does not handle aTRACE_TYPE_PARTIAL_LINEwell and causes the machine to stall.No, if it stalls here, it means it handles it well :)
Okay. As stated earlier, the cause is an incorrect
TRACE_TYPE_PARTIAL_LINEreturn.Quoted 31 lines
Quoted 14 lines
While a
TRACE_TYPE_UNHANDLEDreturn causes the tracer to output unrelated data, aTRACE_TYPE_HANDLEDreturn presents a clean output (minus all partial traces).Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 6 ++++++ 1 file changed, 6 insertions(+)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 9 lines
kernel/trace/trace_ksym.c +6
-0@@ -389,6 +389,12 @@ static enum print_line_t ksym_trace_outptrace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;Yeah that seems a good fix. But such silent ignored traces may hide bugs (current or futures).
Is it a common situation to have a zero ip or an undefined breakpoint type? How can that happen?Quoted 2 lines
Thanks.
When a symbol is removed from ksym's trace list, it causes unregistration of the breakpoints and the breakpoint structure is kfree()-ed.
Now, looking at the relevant code in
ksym_hbp_handler()intrace_ksym.c88 entry = ring_buffer_event_data(event); 89 strlcpy(entry->ksym_name, hbp->info.name, KSYM_SYMBOL_LEN); 90 entry->ksym_hbp = hbp; 91 entry->ip = instruction_pointer(regs); 92 strlcpy(entry->p_name, current->comm, TASK_COMM_LEN);
Quoted 6 lines
While the
ksym_nameandp_namefields are copied into the ring buffer, only a pointer to the structure is provided (in line 90). So, whenksym_hbpis unregistered the fields contained in it, namely 'type' and 'ip' are lost (I'm not sure how the fields beyond 'field->ksym_hbp' is accessible inksym_trace_output()when it is supposed to be already free).
If they are freed, they could still be available if the memory place hasn't been touched by someone else since you kfree'ed it.
But that's buggy and dangerous :)
Quoted 9 lines
This patch would cause an early return when such is the case. I also plan to remove the 'default case' in
ksym_trace_output()as it was meant to handle a zero value for type (other values are eliminated at the time of breakpoint registration itself).Please find a patch below that eliminates the two issues of output concatenation and machine stall, excepting for one issue which I'm unable to reason out.
Quoted 5 lines
A 'cat
trace_pipe' done immediately after removal of an entry (without a preceding 'cat trace') terminates only with a SIGTERM (kill <pid>) and not a SIGINT (Ctrl+C). However, if the trace buffer receives any new data subsequently, it responds to the Ctrl+C signal. I would be glad to receive your suggestions in this regard.
Hmm, weird...
Quoted 5 lines
===================================================================
kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 10 lines
kernel/trace/trace_ksym.c +6
-0@@ -388,6 +388,12 @@ static enum print_line_t ksym_trace_outpreturn TRACE_TYPE_UNHANDLED ;trace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;
No I still think it's dangerous, it may dereference a freed pointer.
Quoted 15 lines
kernel/trace/trace_ksym.c +0
-0ret = trace_seq_printf(s, "%-15s %-5d %-3d %-20s ", field->p_name,
entry->pid,iter->cpu,field->ksym_name);kernel/trace/trace_ksym.c +0
-3@@ -401,10 +407,7 @@ static enum print_line_t ksym_trace_outpcase HW_BREAKPOINT_RW : ret =trace_seq_printf (s," RW " );break ; -default : -return TRACE_TYPE_PARTIAL_LINE ; } -if (!ret)return TRACE_TYPE_PARTIAL_LINE ;
Quoted 3 lines
Thanks,
K.Prasad
What do you think about the full copy I suggested before? That would solve these issues in a simple way and would be _much_ more safe.
If you try this, tell me if you still have such signal issue.
Thanks.
Re: [Patch 2/2] ksym_tracer:Handle machine stall when cat trace_pipe for ksym tracer
Quoted 141 lines · Frederic Weisbecker
Quoted 59 lines · K.Prasad
Quoted 8 lines · Frederic Weisbecker
Quoted 2 lines · K.Prasad
'
trace_pipe' does not handle aTRACE_TYPE_PARTIAL_LINEwell and causes the machine to stall.No, if it stalls here, it means it handles it well :)
Okay. As stated earlier, the cause is an incorrect
TRACE_TYPE_PARTIAL_LINEreturn.Quoted 31 lines
Quoted 14 lines
While a
TRACE_TYPE_UNHANDLEDreturn causes the tracer to output unrelated data, aTRACE_TYPE_HANDLEDreturn presents a clean output (minus all partial traces).Signed-off-by K.Prasad <[email protected]>--- kernel/trace/trace_ksym.c | 6 ++++++ 1 file changed, 6 insertions(+)Index:
linux-2.6-tip.hbkpt/kernel/trace/trace_ksym.c
===================================================================kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 9 lines
kernel/trace/trace_ksym.c +6
-0@@ -389,6 +389,12 @@ static enum print_line_t ksym_trace_outptrace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;Yeah that seems a good fix. But such silent ignored traces may hide bugs (current or futures).
Is it a common situation to have a zero ip or an undefined breakpoint type? How can that happen?Quoted 2 lines
Thanks.
When a symbol is removed from ksym's trace list, it causes unregistration of the breakpoints and the breakpoint structure is kfree()-ed.
Now, looking at the relevant code in
ksym_hbp_handler()intrace_ksym.c88 entry = ring_buffer_event_data(event); 89 strlcpy(entry->ksym_name, hbp->info.name, KSYM_SYMBOL_LEN); 90 entry->ksym_hbp = hbp; 91 entry->ip = instruction_pointer(regs); 92 strlcpy(entry->p_name, current->comm, TASK_COMM_LEN);Quoted 6 lines
While the
ksym_nameandp_namefields are copied into the ring buffer, only a pointer to the structure is provided (in line 90). So, whenksym_hbpis unregistered the fields contained in it, namely 'type' and 'ip' are lost (I'm not sure how the fields beyond 'field->ksym_hbp' is accessible inksym_trace_output()when it is supposed to be already free).If they are freed, they could still be available if the memory place hasn't been touched by someone else since you kfree'ed it.
But that's buggy and dangerous :)
Quoted 9 lines
This patch would cause an early return when such is the case. I also plan to remove the 'default case' in
ksym_trace_output()as it was meant to handle a zero value for type (other values are eliminated at the time of breakpoint registration itself).Please find a patch below that eliminates the two issues of output concatenation and machine stall, excepting for one issue which I'm unable to reason out.
Quoted 5 lines
A 'cat
trace_pipe' done immediately after removal of an entry (without a preceding 'cat trace') terminates only with a SIGTERM (kill <pid>) and not a SIGINT (Ctrl+C). However, if the trace buffer receives any new data subsequently, it responds to the Ctrl+C signal. I would be glad to receive your suggestions in this regard.Hmm, weird...
Quoted 5 lines
===================================================================
kernel/trace/trace_ksym.c static enum print_line_t ksym_trace_outp · line 397
Quoted 10 lines
kernel/trace/trace_ksym.c +6
-0@@ -388,6 +388,12 @@ static enum print_line_t ksym_trace_outpreturn TRACE_TYPE_UNHANDLED ;trace_assign_type (field, entry); +/* +* Return early without any output if we don't have sufficient +* information +*/ +if ((!field->ksym_hbp->info.type) || (!field->ip)) +return TRACE_TYPE_HANDLED ;No I still think it's dangerous, it may dereference a freed pointer.
Quoted 15 lines
kernel/trace/trace_ksym.c +0
-0ret = trace_seq_printf(s, "%-15s %-5d %-3d %-20s ", field->p_name,
entry->pid,iter->cpu,field->ksym_name);kernel/trace/trace_ksym.c +0
-3@@ -401,10 +407,7 @@ static enum print_line_t ksym_trace_outpcase HW_BREAKPOINT_RW : ret =trace_seq_printf (s," RW " );break ; -default : -return TRACE_TYPE_PARTIAL_LINE ; } -if (!ret)return TRACE_TYPE_PARTIAL_LINE ;Quoted 3 lines
Thanks,
K.PrasadWhat do you think about the full copy I suggested before? That would solve these issues in a simple way and would be _much_ more safe.
Quoted 4 lines
If you try this, tell me if you still have such signal issue.
Thanks.
I introduced a new structure to contain the data for the ring buffer and did a copy operation. But I am able to still re-create the issue. Here's a patch that does a copy of the trace data (when a breakpoint exception is hit).
I will formally send this patch for inclusion into -tip tree after some more testing (and yes, it indeed solves the problem of output concatenation and machine stall!).
kernel/trace/trace_ksym.c +4 -7
@@ -71,7 +71,7 @@ void ksym_hbp_handler(struct hw_breakpoi {struct ring_buffer_event *event;struct trace_array *tr; -struct trace_ksym *entry; +struct trace_ksym_rb *entry;int pc;if (!ksym_tracing_enabled)
@@ -87,7 +87,7 @@ void ksym_hbp_handler(struct hw_breakpoi entry =ring_buffer_event_data (event);strlcpy (entry->ksym_name, hbp->info.name,KSYM_SYMBOL_LEN ); - entry->ksym_hbp = hbp; +memcpy (&(entry->ksym_hbp), hbp,sizeof (struct hw_breakpoint )); entry->ip =instruction_pointer (regs);strlcpy (entry->p_name, current->comm,TASK_COMM_LEN );#ifdef CONFIG_PROFILE_KSYM_TRACER
@@ -380,7 +380,7 @@ static enum print_line_t ksym_trace_outp {struct trace_entry *entry = iter->ent;struct trace_seq *s = &iter->seq; -struct trace_ksym *field; +struct trace_ksym_rb *field;char str[KSYM_SYMBOL_LEN ];int ret;
@@ -394,17 +394,14 @@ static enum print_line_t ksym_trace_outpif (!ret)return TRACE_TYPE_PARTIAL_LINE ; -switch (field->ksym_hbp->info.type) { +switch (field->ksym_hbp.info.type) {case HW_BREAKPOINT_WRITE : ret =trace_seq_printf (s," W " );break ;case HW_BREAKPOINT_RW : ret =trace_seq_printf (s," RW " );break ; -default : -return TRACE_TYPE_PARTIAL_LINE ; } -if (!ret)return TRACE_TYPE_PARTIAL_LINE ;
kernel/trace/trace.h +9 -3
@@ -216,15 +216,21 @@ struct syscall_trace_exit {extern int process_new_ksym_entry (char *ksymname,int op,unsigned long addr);struct trace_ksym { -struct trace_entry ent;struct hw_breakpoint *ksym_hbp;unsigned long ksym_addr; -unsigned long ip;#ifdef CONFIG_PROFILE_KSYM_TRACER unsigned long counter;#endif struct hlist_node ksym_hlist;char ksym_name[KSYM_NAME_LEN ]; +}; + +/* Ring buffer's copy of the breakpoint data */ +struct trace_ksym_rb { +struct trace_entry ent; +struct hw_breakpoint ksym_hbp; +unsigned long ip; +char ksym_name[KSYM_NAME_LEN ];char p_name[TASK_COMM_LEN ]; };
@@ -343,7 +349,7 @@ extern void __ftrace_bad_type(void); TRACE_SYSCALL_ENTER); \ IF_ASSIGN(var, ent, struct syscall_trace_exit, \TRACE_SYSCALL_EXIT ); \ - IF_ASSIGN(var, ent, struct trace_ksym, TRACE_KSYM); \ + IF_ASSIGN(var, ent, struct trace_ksym_rb, TRACE_KSYM); \__ftrace_bad_type (); \ }while (0 )