trace-cmd profile enhancements
cover trace-cmd profile enhancements
I had to use trace-cmd profile to debug a regression with hhvm recently. These are the patches I needed to make trace-cmd profile useful to me. There are a few changes in here
1) Adding --merge to trace-cmd profile. This merges events for like comms together. HHVM is a huge app that spawns worker threads to handle requests, I needed all of this data in one place to see how the app as a whole was behaving, --merge provides this.
2) Fixup the output formatting. I also wanted to see which syscalls were taking the longest but this was hard because we were just spitting out the syscall number and it wasn't in any order, so now all the events are ordered based on time taken and the syscall names are printed instead of the number.
3) Random trace-cmd hist fix, because it didn't work right with instances.
I've been using these in production for a couple of weeks now and they seem to work well. Thanks,
Josef
PATCH 1/5 trace-cmd: add option to group like comms for profile
When profiling large applications that have multiple children we really want to see the profile of the app all together. Add a --merge option to accomplish this for profile. This matches events based on comm instead of PID, this way we can easilly see what a giant app is doing, such as HHVM. Thanks,
--- trace-hash-local.h | 7 ++ trace-local.h | 1 + trace-profile.c | 207 +++++++++++++++++++++++++++++++++++++++++++++++++---- trace-record.c | 5 ++ trace-usage.c | 1 + 5 files changed, 208 insertions(+), 13 deletions(-)
trace-hash-local.h +7 -0
@@ -48,4 +48,11 @@ static inline unsigned int trace_hash(int val) return hash; } +static inline unsigned int trace_hash_str (char *str) +{ +int val =0 ; + +while (*str) val += (int )*(str++); +return trace_hash (val); +}#endif /* _TRACE_HASH_LOCAL_H */
trace-local.h +1 -0
@@ -81,6 +81,7 @@ int trace_profile_record(struct tracecmd_input *handle,void trace_init_profile (struct tracecmd_input *handle,struct hook_list *hooks,int global);int trace_profile (void ); +void trace_profile_merge (void );struct tracecmd_input * trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,
trace-profile.c +194 -13
339 lines not on this page · raw message
trace-record.c +5 -0
@@ -3703,6 +3703,7 @@ static void add_hook(struct buffer_instance *instance, const char *arg) }enum { + OPT_merge =250 , OPT_stderr =251 , OPT_profile =252 , OPT_nosplice =253 ,
@@ -3871,6 +3872,7 @@ void trace_record (int argc, char **argv) {"nosplice", no_argument, NULL, OPT_nosplice}, {"profile", no_argument, NULL, OPT_profile}, {"stderr", no_argument, NULL, OPT_stderr}, + {"merge", no_argument, NULL, OPT_merge}, {"help", no_argument, NULL, '?'}, {NULL, 0, NULL, 0} };
@@ -4102,6 +4104,9 @@ void trace_record (int argc, char **argv)close (1 );dup2 (2 ,1 );break ; +case OPT_merge: +trace_profile_merge (); +break ;default :usage (argv); }
trace-usage.c +1 -0
@@ -48,6 +48,7 @@ static struct usage_help usage_help[] = {" -B create sub buffer and folling events will be enabled here\n" " -k do not reset the buffers after tracing.\n" " -i do not fail if an event is not found\n" +" --merge used with --profile, merge events for related comms\n" " --profile enable tracing options needed for report --profile\n" " --func-stack perform a stack trace for function tracer\n" " (use with caution)\n"
Signature · 1 line
2.1.0
Re: [PATCH 1/5] trace-cmd: add option to group like comms for profile
Finally have some time to look at this.
Quoted 11 lines · On Fri, 8 May 2015 15:29:00 -0400 Josef Bacik <[email protected]> wrote:
When profiling large applications that have multiple children we really want to see the profile of the app all together. Add a --merge option to accomplish this for profile. This matches events based on comm instead of PID, this way we can easilly see what a giant app is doing, such as HHVM. Thanks,
Signed-off-by Josef Bacik <[email protected]>--- trace-hash-local.h | 7 ++ trace-local.h | 1 + trace-profile.c | 207 +++++++++++++++++++++++++++++++++++++++++++++++++---- trace-record.c | 5 ++
Shouldn't we add this option to trace-cmd report --profile ?
Also, I think we need a man page.
Also, I'm not so sure I like the "--merge" name. It's not very descriptive to what it does. What about "--by-comm"? Or maybe "--comm"?
Quoted 8 lines
trace-usage.c | 1 + 5 files changed, 208 insertions(+), 13 deletions(-)trace-hash-local.h +0
-0
trace-hash-local.h static inline unsigned int trace_hash(int val) · line 57
Quoted 10 lines
trace-hash-local.h +7
-0@@ -48,4 +48,11 @@ static inline unsigned int trace_hash(int val)return hash; } +static inline unsigned int trace_hash_str (char *str) +{ +int val =0 ; + +while (*str) val += (int )*(str++); +return trace_hash (val); +}
Hmm, the above doesn't account for order of characters. That is "bash" is the same as "shab". What about this:
int val = 0; int i; for (i = 0; str[i]; i++) val += ((int)str[i]) << (i & 0xffffff); return trace_hash(val);
Quoted 210 lines
trace-hash-local.h +0
-0#endif /* _TRACE_HASH_LOCAL_H */ trace-local.h +1
-0@@ -81,6 +81,7 @@ int trace_profile_record(struct tracecmd_input *handle,void trace_init_profile (struct tracecmd_input *handle,struct hook_list *hooks,int global);int trace_profile (void ); +void trace_profile_merge (void );struct tracecmd_input * trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,trace-profile.c +80
-8@@ -34,6 +34,7 @@#define start_from_item (item) container_of(item, struct start_data, hash)#define event_from_item (item) container_of(item, struct event_hash, hash)#define stack_from_item (item) container_of(item, struct stack_data, hash) +#define group_from_item (item) container_of(item, struct group_data, hash)#define event_data_from_item (item) container_of(item, struct event_data, hash)static unsigned long long nsecs_per_sec (unsigned long long ts)@@ -141,6 +142,12 @@ struct event_hash {struct trace_hash stacks; }; +struct group_data { +struct trace_hash_item hash; +char *comm; +struct trace_hash event_hash; +}; +struct task_data {struct trace_hash_item hash;int pid;@@ -156,6 +163,7 @@ struct task_data {struct event_hash *last_event;struct pevent_record *last_stack;struct handle_data *handle; +struct group_data *group; };struct cpu_info {@@ -173,6 +181,7 @@ struct handle_data {struct pevent *pevent;struct trace_hash events; +struct trace_hash group_hash;struct cpu_info **cpu_data;@@ -196,6 +205,12 @@ struct handle_data {static struct handle_data *handles;static struct event_data *stacktrace_event; +static bool merge_like_comms = false; + +void trace_profile_merge (void ) +{ + merge_like_comms = true; +}static struct start_data * add_start(struct task_data *task,@@ -250,6 +265,7 @@ static struct event_hash * find_event_hash(struct task_data *task, struct event_data_match *edata) { struct event_hash *event_hash; +struct trace_hash *hash;struct trace_hash_item *item;unsigned long long key;@@ -257,7 +273,11 @@ find_event_hash(struct task_data *task, struct event_data_match *edata) (unsigned long )edata->search_val + (unsigned long )edata->val; key =trace_hash (key); - item =trace_hash_find (&task->event_hash, key, match_event, edata); +if (task->group) + hash = &task->group->event_hash; +else + hash = &task->event_hash; + item =trace_hash_find (hash, key, match_event, edata);if (item)return event_from_item (item);@@ -270,7 +290,7 @@ find_event_hash(struct task_data *task, struct event_data_match *edata) event_hash->hash.key = key;trace_hash_init (&event_hash->stacks,32 ); -trace_hash_add (&task->event_hash, &event_hash->hash); +trace_hash_add (hash, &event_hash->hash);return event_hash; }@@ -496,9 +516,59 @@ find_task(struct handle_data *h, int pid)return last_task; } +static int match_group (struct trace_hash_item *item,void *data) +{ +struct group_data *group =group_from_item (item); + +return strcmp (group->comm, (char *)data) ==0 ; +} + +static void add_group (struct handle_data *h,struct task_data *task) +{ +unsigned long long key; +struct trace_hash_item *item; +void *data = task->comm; + +if (!merge_like_comms) +return ; +if (task->group || !task->comm) +return ; + + key =trace_hash_str (task->comm); + + item =trace_hash_find (&h->group_hash, key, match_group, data); +if (item) { + task->group =group_from_item (item); + }else { +struct group_data *grp; +struct trace_hash_item **bucket; + + grp =malloc_or_die (sizeof (*grp)); +memset (grp,0 ,sizeof (*grp)); + + grp->comm =strdup (task->comm); +if (!grp->comm) +die ("strdup" ); + grp->hash.key = key; +trace_hash_add (&h->group_hash, &grp->hash); + task->group = grp; +trace_hash_init (&grp->event_hash,32 ); + +trace_hash_for_each_bucket (bucket, &task->event_hash) { + trace_hash_while_item(item, bucket) { +struct event_hash *event_hash; + + event_hash =event_from_item (item); +trace_hash_del (&event_hash->hash); +trace_hash_add (&grp->event_hash, &event_hash->hash); + } + } + } +} +static void -add_task_comm (struct task_data *task,struct format_field *field, -struct pevent_record *record) +add_task_comm (struct handle_data *h,struct task_data *task, +struct format_field *field,struct pevent_record *record) {const char *comm;@@ -506,6 +576,7 @@ add_task_comm(struct task_data *task, struct format_field *field, comm = record->data + field->offset;memcpy (task->comm, comm, field->size); task->comm[field->size] =0 ; +add_group (h, task); }/* Account for tasks that don't have starts */ @@ -932,7 +1003,7 @@ static int handle_sched_switch_event(struct handle_data *h, task =find_task (h, prev_pid);if (!task->comm) -add_task_comm (task, h->switch_prev_comm, record); +add_task_comm (h, task, h->switch_prev_comm, record);if (prev_state) task->sleeping =1 ;@@ -940,13 +1011,13 @@ static int handle_sched_switch_event(struct handle_data *h, task->sleeping =0 ;/* task is being scheduled out. prev_state tells why */ - start =add_start (task, event_data, record, prev_pid, prev_state); + start =add_start (task, event_data, record, prev_state, prev_state); task->last_start = start; task->last_event =NULL ; task =find_task (h, next_pid);if (!task->comm) -add_task_comm (task, h->switch_next_comm, record); +add_task_comm (h, task, h->switch_next_comm, record);/* * If the next task was blocked, it required a wakeup to @@ -1133,7 +1204,7 @@ static int handle_sched_wakeup_event(struct handle_data *h, task =find_task (h, pid);if (!task->comm) -add_task_comm (task, h->wakeup_comm, record); +add_task_comm (h, task, h->wakeup_comm, record);/* if the task isn't sleeping, then ignore the wake up */ if (!task->sleeping) {@@ -1196,6 +1267,7 @@ void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hook,trace_hash_init (&h->task_hash,1024 );trace_hash_init (&h->events,1024 ); +trace_hash_init (&h->group_hash,512 ); h->handle = handle; h->pevent = pevent;
trace-profile.c static void output_task(struct handle_data *h, struct task_data *task) · line 1951
Quoted 6 lines
trace-profile.c +3
-0@@ -1874,6 +1946,9 @@ static void output_task(struct handle_data *h, struct task_data *task)int nr_events =0 ;int i; +if (trace_hash_empty (&task->event_hash)) +return ; +
Was this needed for this patch, or is it for groups? Even if the task doesn't have any events, I do sometimes want to see if it was recorder.
If you added this to not show tasks if they belong to a group, why not add:
if (task->group) return;
?
Quoted 148 lines
trace-profile.c +101
-5if (task->comm) comm = task->comm;else @@ -1907,6 +1982,39 @@ static void output_task(struct handle_data *h, struct task_data *task)free (events); } +static void output_group (struct handle_data *h,struct group_data *group) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct event_hash **events; +int nr_events =0 ; +int i; + +printf ("\ngroup: %s\n" , group->comm); + +trace_hash_for_each_bucket (bucket, &group->event_hash) { +trace_hash_for_each_item (item, bucket) { + nr_events++; + } + } + + events =malloc_or_die (sizeof (*events) * nr_events); + + i =0 ; +trace_hash_for_each_bucket (bucket, &group->event_hash) { +trace_hash_for_each_item (item, bucket) { + events[i++] =event_from_item (item); + } + } + +qsort (events, nr_events,sizeof (*events), compare_events); + +for (i =0 ; i < nr_events; i++) +output_event (events[i]); + +free (events); +} +static int compare_tasks (const void *a,const void *b) {struct task_data *const *A = a;@@ -1919,6 +2027,14 @@ static int compare_tasks(const void *a, const void *b)return 0 ; } +static int compare_groups (const void *a,const void *b) +{ +const char *A = a; +const char *B = b; + +return strcmp (A ,B ); +} +static void free_event_hash (struct event_hash *event_hash) {struct trace_hash_item **bucket;@@ -1976,6 +2092,25 @@ static void free_task(struct task_data *task)free (task); } +static void free_group (struct group_data *group) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct event_hash *event_hash; + +free (group->comm); + +trace_hash_for_each_bucket (bucket, &group->event_hash) { + trace_hash_while_item(item, bucket) { + event_hash =event_from_item (item); +trace_hash_del (item); +free_event_hash (event_hash); + } + } +trace_hash_free (&group->event_hash); +free (group); +} +static void show_global_task (struct handle_data *h,struct task_data *task) {@@ -1985,7 +2120,7 @@ static void show_global_task(struct handle_data *h,output_task (h, task); } -static void output_handle (struct handle_data *h) +static void output_tasks (struct handle_data *h) {struct trace_hash_item **bucket;struct trace_hash_item *item;@@ -1993,10 +2128,6 @@ static void output_handle(struct handle_data *h)int nr_tasks =0 ;int i; -show_global_task (h, h->global_task); -for (i =0 ; i < h->cpus; i++) -show_global_task (h, &h->global_percpu_tasks[i]); -trace_hash_for_each_bucket (bucket, &h->task_hash) {trace_hash_for_each_item (item, bucket) { nr_tasks++;@@ -2024,6 +2155,56 @@ static void output_handle(struct handle_data *h)free (tasks); } +static void output_groups (struct handle_data *h) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct group_data **groups; +int nr_groups =0 ; +int i; + +trace_hash_for_each_bucket (bucket, &h->group_hash) { +trace_hash_for_each_item (item, bucket) { + nr_groups++; + } + } + +if (nr_groups ==0 ) +return ; + + groups =malloc_or_die (sizeof (*groups) * nr_groups); + + nr_groups =0 ; + +trace_hash_for_each_bucket (bucket, &h->group_hash) { + trace_hash_while_item(item, bucket) { + groups[nr_groups++] =group_from_item (item); +trace_hash_del (item); + } + } + +qsort (groups, nr_groups,sizeof (*groups), compare_groups); + +for (i =0 ; i < nr_groups; i++) { +output_group (h, groups[i]); +free_group (groups[i]); + } + +free (groups); +} + +static void output_handle (struct handle_data *h) +{
trace-profile.c static void output_handle(struct handle_data *h) · line 2207
Quoted 10 lines
trace-profile.c +10
-0+int i; + +show_global_task (h, h->global_task); +for (i =0 ; i < h->cpus; i++) +show_global_task (h, &h->global_percpu_tasks[i]); + +output_groups (h); +output_tasks (h); +} +
Hmm, I wonder if the combination should be done on output. Because how do you handle threads where an event is per task? Like scheduling one out and the other one in? What about wake ups? If a bunch of threads get woken up, how do we match the wakeups to the tasks?
Would it be easier to just match all the tasks by comm after the recording was done? I would think matching pids would be quicker than matching strings.
-- Steve
Quoted 45 lines
trace-profile.c +0
-0int trace_profile (void ) {struct handle_data *h;trace-record.c +5
-0@@ -3703,6 +3703,7 @@ static void add_hook(struct buffer_instance *instance, const char *arg) }enum { + OPT_merge =250 , OPT_stderr =251 , OPT_profile =252 , OPT_nosplice =253 ,@@ -3871,6 +3872,7 @@ void trace_record (int argc, char **argv) {"nosplice", no_argument, NULL, OPT_nosplice}, {"profile", no_argument, NULL, OPT_profile}, {"stderr", no_argument, NULL, OPT_stderr}, + {"merge", no_argument, NULL, OPT_merge}, {"help", no_argument, NULL, '?'}, {NULL, 0, NULL, 0} };@@ -4102,6 +4104,9 @@ void trace_record (int argc, char **argv)close (1 );dup2 (2 ,1 );break ; +case OPT_merge: +trace_profile_merge (); +break ;default :usage (argv); }trace-usage.c +1
-0@@ -48,6 +48,7 @@ static struct usage_help usage_help[] = {" -B create sub buffer and folling events will be enabled here\n" " -k do not reset the buffers after tracing.\n" " -i do not fail if an event is not found\n" +" --merge used with --profile, merge events for related comms\n" " --profile enable tracing options needed for report --profile\n" " --func-stack perform a stack trace for function tracer\n" " (use with caution)\n"
Re: [PATCH 1/5] trace-cmd: add option to group like comms for profile
Quoted 11 lines · Steven Rostedt
Finally have some time to look at this.
Quoted 5 lines · On Fri, 8 May 2015 15:29:00 -0400 Josef Bacik <[email protected]> wrote:
When profiling large applications that have multiple children we really want to see the profile of the app all together. Add a --merge option to accomplish this for profile. This matches events based on comm instead of PID, this way we can easilly see what a giant app is doing, such as HHVM. Thanks,
Quoted 8 lines
Quoted 6 lines
Signed-off-by Josef Bacik <[email protected]>--- trace-hash-local.h | 7 ++ trace-local.h | 1 + trace-profile.c | 207 +++++++++++++++++++++++++++++++++++++++++++++++++---- trace-record.c | 5 ++Shouldn't we add this option to trace-cmd report --profile ?
Ah yeah I missed that part of it, I'll add that bit.
Quoted 3 lines
Also, I think we need a man page.
No complaints here, but I'm not volunteering either ;).
Quoted 3 lines
Also, I'm not so sure I like the "--merge" name. It's not very descriptive to what it does. What about "--by-comm"? Or maybe "--comm"?
I like --by-comm, I'll change it.
Quoted 22 lines
Quoted 8 lines
trace-usage.c | 1 + 5 files changed, 208 insertions(+), 13 deletions(-)trace-hash-local.h +0
-0trace-hash-local.h static inline unsigned int trace_hash(int val) · line 57
Quoted 10 lines
trace-hash-local.h +7
-0@@ -48,4 +48,11 @@ static inline unsigned int trace_hash(int val)return hash; } +static inline unsigned int trace_hash_str (char *str) +{ +int val =0 ; + +while (*str) val += (int )*(str++); +return trace_hash (val); +}Hmm, the above doesn't account for order of characters. That is "bash" is the same as "shab". What about this:
Quoted 7 lines
int val = 0; int i; for (i = 0; str[i]; i++) val += ((int)str[i]) << (i & 0xffffff); return trace_hash(val);
Good point, I meant to look up a real hashing thing later but obviously forgot, I'll switch to yours.
Quoted 222 lines
Quoted 212 lines
trace-hash-local.h +0
-0#endif /* _TRACE_HASH_LOCAL_H */ trace-local.h +1
-0@@ -81,6 +81,7 @@ int trace_profile_record(struct tracecmd_input *handle,void trace_init_profile (struct tracecmd_input *handle,struct hook_list *hooks,int global);int trace_profile (void ); +void trace_profile_merge (void );struct tracecmd_input * trace_stream_init(struct buffer_instance *instance, int cpu, int fd, int cpus,trace-profile.c +80
-8@@ -34,6 +34,7 @@#define start_from_item (item) container_of(item, struct start_data, hash)#define event_from_item (item) container_of(item, struct event_hash, hash)#define stack_from_item (item) container_of(item, struct stack_data, hash) +#define group_from_item (item) container_of(item, struct group_data, hash)#define event_data_from_item (item) container_of(item, struct event_data, hash)static unsigned long long nsecs_per_sec (unsigned long long ts)@@ -141,6 +142,12 @@ struct event_hash {struct trace_hash stacks; }; +struct group_data { +struct trace_hash_item hash; +char *comm; +struct trace_hash event_hash; +}; +struct task_data {struct trace_hash_item hash;int pid;@@ -156,6 +163,7 @@ struct task_data {struct event_hash *last_event;struct pevent_record *last_stack;struct handle_data *handle; +struct group_data *group; };struct cpu_info {@@ -173,6 +181,7 @@ struct handle_data {struct pevent *pevent;struct trace_hash events; +struct trace_hash group_hash;struct cpu_info **cpu_data;@@ -196,6 +205,12 @@ struct handle_data {static struct handle_data *handles;static struct event_data *stacktrace_event; +static bool merge_like_comms = false; + +void trace_profile_merge (void ) +{ + merge_like_comms = true; +}static struct start_data * add_start(struct task_data *task,@@ -250,6 +265,7 @@ static struct event_hash * find_event_hash(struct task_data *task, struct event_data_match *edata) { struct event_hash *event_hash; +struct trace_hash *hash;struct trace_hash_item *item;unsigned long long key;@@ -257,7 +273,11 @@ find_event_hash(struct task_data *task, struct event_data_match *edata) (unsigned long )edata->search_val + (unsigned long )edata->val; key =trace_hash (key); - item =trace_hash_find (&task->event_hash, key, match_event, edata); +if (task->group) + hash = &task->group->event_hash; +else + hash = &task->event_hash; + item =trace_hash_find (hash, key, match_event, edata);if (item)return event_from_item (item);@@ -270,7 +290,7 @@ find_event_hash(struct task_data *task, struct event_data_match *edata) event_hash->hash.key = key;trace_hash_init (&event_hash->stacks,32 ); -trace_hash_add (&task->event_hash, &event_hash->hash); +trace_hash_add (hash, &event_hash->hash);return event_hash; }@@ -496,9 +516,59 @@ find_task(struct handle_data *h, int pid)return last_task; } +static int match_group (struct trace_hash_item *item,void *data) +{ +struct group_data *group =group_from_item (item); + +return strcmp (group->comm, (char *)data) ==0 ; +} + +static void add_group (struct handle_data *h,struct task_data *task) +{ +unsigned long long key; +struct trace_hash_item *item; +void *data = task->comm; + +if (!merge_like_comms) +return ; +if (task->group || !task->comm) +return ; + + key =trace_hash_str (task->comm); + + item =trace_hash_find (&h->group_hash, key, match_group, data); +if (item) { + task->group =group_from_item (item); + }else { +struct group_data *grp; +struct trace_hash_item **bucket; + + grp =malloc_or_die (sizeof (*grp)); +memset (grp,0 ,sizeof (*grp)); + + grp->comm =strdup (task->comm); +if (!grp->comm) +die ("strdup" ); + grp->hash.key = key; +trace_hash_add (&h->group_hash, &grp->hash); + task->group = grp; +trace_hash_init (&grp->event_hash,32 ); + +trace_hash_for_each_bucket (bucket, &task->event_hash) { + trace_hash_while_item(item, bucket) { +struct event_hash *event_hash; + + event_hash =event_from_item (item); +trace_hash_del (&event_hash->hash); +trace_hash_add (&grp->event_hash, &event_hash->hash); + } + } + } +} +static void -add_task_comm (struct task_data *task,struct format_field *field, -struct pevent_record *record) +add_task_comm (struct handle_data *h,struct task_data *task, +struct format_field *field,struct pevent_record *record) {const char *comm;@@ -506,6 +576,7 @@ add_task_comm(struct task_data *task, struct format_field *field, comm = record->data + field->offset;memcpy (task->comm, comm, field->size); task->comm[field->size] =0 ; +add_group (h, task); }/* Account for tasks that don't have starts */ @@ -932,7 +1003,7 @@ static int handle_sched_switch_event(struct handle_data *h, task =find_task (h, prev_pid);if (!task->comm) -add_task_comm (task, h->switch_prev_comm, record); +add_task_comm (h, task, h->switch_prev_comm, record);if (prev_state) task->sleeping =1 ;@@ -940,13 +1011,13 @@ static int handle_sched_switch_event(struct handle_data *h, task->sleeping =0 ;/* task is being scheduled out. prev_state tells why */ - start =add_start (task, event_data, record, prev_pid, prev_state); + start =add_start (task, event_data, record, prev_state, prev_state); task->last_start = start; task->last_event =NULL ; task =find_task (h, next_pid);if (!task->comm) -add_task_comm (task, h->switch_next_comm, record); +add_task_comm (h, task, h->switch_next_comm, record);/* * If the next task was blocked, it required a wakeup to @@ -1133,7 +1204,7 @@ static int handle_sched_wakeup_event(struct handle_data *h, task =find_task (h, pid);if (!task->comm) -add_task_comm (task, h->wakeup_comm, record); +add_task_comm (h, task, h->wakeup_comm, record);/* if the task isn't sleeping, then ignore the wake up */ if (!task->sleeping) {@@ -1196,6 +1267,7 @@ void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hook,trace_hash_init (&h->task_hash,1024 );trace_hash_init (&h->events,1024 ); +trace_hash_init (&h->group_hash,512 ); h->handle = handle; h->pevent = pevent;trace-profile.c static void output_task(struct handle_data *h, struct task_data *task) · line 1951
Quoted 6 lines
trace-profile.c +3
-0@@ -1874,6 +1946,9 @@ static void output_task(struct handle_data *h, struct task_data *task)int nr_events =0 ;int i; +if (trace_hash_empty (&task->event_hash)) +return ; +Was this needed for this patch, or is it for groups? Even if the task doesn't have any events, I do sometimes want to see if it was recorder.
Quoted 6 lines
If you added this to not show tasks if they belong to a group, why not add:
if (task->group) return;?
Yup I can do that too, I added it because we got a bunch of "task: blah" output at the end using groups and it seemed ugly, so this check made it only output tasks that weren't grouped. I'll make your change instead.
Quoted 157 lines
Quoted 157 lines
trace-profile.c +109
-5if (task->comm) comm = task->comm;else @@ -1907,6 +1982,39 @@ static void output_task(struct handle_data *h, struct task_data *task)free (events); } +static void output_group (struct handle_data *h,struct group_data *group) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct event_hash **events; +int nr_events =0 ; +int i; + +printf ("\ngroup: %s\n" , group->comm); + +trace_hash_for_each_bucket (bucket, &group->event_hash) { +trace_hash_for_each_item (item, bucket) { + nr_events++; + } + } + + events =malloc_or_die (sizeof (*events) * nr_events); + + i =0 ; +trace_hash_for_each_bucket (bucket, &group->event_hash) { +trace_hash_for_each_item (item, bucket) { + events[i++] =event_from_item (item); + } + } + +qsort (events, nr_events,sizeof (*events), compare_events); + +for (i =0 ; i < nr_events; i++) +output_event (events[i]); + +free (events); +} +static int compare_tasks (const void *a,const void *b) {struct task_data *const *A = a;@@ -1919,6 +2027,14 @@ static int compare_tasks(const void *a, const void *b)return 0 ; } +static int compare_groups (const void *a,const void *b) +{ +const char *A = a; +const char *B = b; + +return strcmp (A ,B ); +} +static void free_event_hash (struct event_hash *event_hash) {struct trace_hash_item **bucket;@@ -1976,6 +2092,25 @@ static void free_task(struct task_data *task)free (task); } +static void free_group (struct group_data *group) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct event_hash *event_hash; + +free (group->comm); + +trace_hash_for_each_bucket (bucket, &group->event_hash) { + trace_hash_while_item(item, bucket) { + event_hash =event_from_item (item); +trace_hash_del (item); +free_event_hash (event_hash); + } + } +trace_hash_free (&group->event_hash); +free (group); +} +static void show_global_task (struct handle_data *h,struct task_data *task) {@@ -1985,7 +2120,7 @@ static void show_global_task(struct handle_data *h,output_task (h, task); } -static void output_handle (struct handle_data *h) +static void output_tasks (struct handle_data *h) {struct trace_hash_item **bucket;struct trace_hash_item *item;@@ -1993,10 +2128,6 @@ static void output_handle(struct handle_data *h)int nr_tasks =0 ;int i; -show_global_task (h, h->global_task); -for (i =0 ; i < h->cpus; i++) -show_global_task (h, &h->global_percpu_tasks[i]); -trace_hash_for_each_bucket (bucket, &h->task_hash) {trace_hash_for_each_item (item, bucket) { nr_tasks++;@@ -2024,6 +2155,56 @@ static void output_handle(struct handle_data *h)free (tasks); } +static void output_groups (struct handle_data *h) +{ +struct trace_hash_item **bucket; +struct trace_hash_item *item; +struct group_data **groups; +int nr_groups =0 ; +int i; + +trace_hash_for_each_bucket (bucket, &h->group_hash) { +trace_hash_for_each_item (item, bucket) { + nr_groups++; + } + } + +if (nr_groups ==0 ) +return ; + + groups =malloc_or_die (sizeof (*groups) * nr_groups); + + nr_groups =0 ; + +trace_hash_for_each_bucket (bucket, &h->group_hash) { + trace_hash_while_item(item, bucket) { + groups[nr_groups++] =group_from_item (item); +trace_hash_del (item); + } + } + +qsort (groups, nr_groups,sizeof (*groups), compare_groups); + +for (i =0 ; i < nr_groups; i++) { +output_group (h, groups[i]); +free_group (groups[i]); + } + +free (groups); +} + +static void output_handle (struct handle_data *h) +{ +int i; + +show_global_task (h, h->global_task); +for (i =0 ; i < h->cpus; i++) +show_global_task (h, &h->global_percpu_tasks[i]); + +output_groups (h); +output_tasks (h);
Quoted 8 lines
trace-profile.c static void output_handle(struct handle_data *h) · line 2207
Quoted 2 lines
trace-profile.c +2
-0+} +Hmm, I wonder if the combination should be done on output. Because how do you handle threads where an event is per task? Like scheduling one out and the other one in? What about wake ups? If a bunch of threads get woken up, how do we match the wakeups to the tasks?
So the scheduling things are all tracked per task. I'm not actually killing the tasks, I'm adding a grouping with all of the per task information collated.
So for example, the schedule wakeup stuff gets handled the same way in the per task hash bucket, and then when it completes and we add it to the event list we go ahead and move it to the group stuff, after we've done all of our accounting.
Quoted 5 lines
Would it be easier to just match all the tasks by comm after the recording was done? I would think matching pids would be quicker than matching strings.
We could move it to the end I guess, I did it this way so there wasn't a bunch of lag between stopping the profile and then outputting stuff because we have to walk all of the tasks and collate its info at that point. Do you feel strongly either way? I'll change it if you do, but otherwise this seems to work fine and doesn't mess up the accounting since the per-pid stuff is tracked in the task itself. Thanks,
Josef
PATCH 2/5 trace-cmd: group schedule stacktraces together with --merge
In reply to Josef BacikWith --merge we want to be able to merge the scheduling information as well so we can see how the whole app spent its time. This is a little tricky because we have to sort based on comm instead of pid. It works out well enough because we store the intermediate information in the per task hash, but we want to group the final information together.
--- trace-profile.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
trace-profile.c +18 -4
@@ -985,6 +985,7 @@ static int handle_sched_switch_event(struct handle_data *h,unsigned long long prev_pid;unsigned long long prev_state;unsigned long long next_pid; +unsigned long long val =0 ;struct start_data *start;/* pid_field holds prev_pid, data_field holds prev_state */
@@ -1019,6 +1020,12 @@ static int handle_sched_switch_event(struct handle_data *h,if (!task->comm)add_task_comm (h, task, h->switch_next_comm, record); +if (task->comm && task->group) { +char *str = task->comm; +while (*str) val += (int )*(str++); + }else + val = next_pid; +/* * If the next task was blocked, it required a wakeup to * restart, and there should be one.
@@ -1028,10 +1035,10 @@ static int handle_sched_switch_event(struct handle_data *h,* * event_data->start holds the sched_wakeup event data. */ -find_and_update_start (task, event_data->start, record->ts, next_pid); +find_and_update_start (task, event_data->start, record->ts, val);/* Look for this task if it was preempted (no wakeup found). */ -find_and_update_start (task, event_data, record->ts, next_pid); +find_and_update_start (task, event_data, record->ts, val);return 0 ; }
@@ -1186,6 +1193,7 @@ static int handle_sched_wakeup_event(struct handle_data *h,struct task_data *task =NULL ;struct start_data *start;unsigned long long success; +unsigned long long val =0 ; proxy =find_task (h, pid);
@@ -1224,11 +1232,17 @@ static int handle_sched_wakeup_event(struct handle_data *h,*/ proxy->proxy = task; +if (task->comm && task->group) { +char *str = task->comm; +while (*str) val += (int )*(str++); + }else + val = pid; +/* There should be a blocked schedule out of this task */ -find_and_update_start (task, event_data->start, record->ts, pid); +find_and_update_start (task, event_data->start, record->ts, val);/* Set this up for timing how long the wakeup takes */ - start =add_start (task, event_data, record, pid, pid); + start =add_start (task, event_data, record, val, val); task->last_event =NULL ; task->last_start = start;
Signature · 1 line
2.1.0
Re: [PATCH 2/5] trace-cmd: group schedule stacktraces together with --merge
Quoted 5 lines · On Fri, 8 May 2015 15:29:01 -0400 Josef Bacik <[email protected]> wrote:
With --merge we want to be able to merge the scheduling information as well so we can see how the whole app spent its time. This is a little tricky because we have to sort based on comm instead of pid. It works out well enough because we store the intermediate information in the per task hash, but we want to group the final information together.
If we do the merge at the end, we wont need to worry about this.
-- Steve
Quoted 3 lines
Signed-off-by Josef Bacik <[email protected]>
Re: [PATCH 2/5] trace-cmd: group schedule stacktraces together with --merge
Quoted 10 lines · Steven Rostedt
Quoted 5 lines · On Fri, 8 May 2015 15:29:01 -0400 Josef Bacik <[email protected]> wrote:
With --merge we want to be able to merge the scheduling information as well so we can see how the whole app spent its time. This is a little tricky because we have to sort based on comm instead of pid. It works out well enough because we store the intermediate information in the per task hash, but we want to group the final information together.
If we do the merge at the end, we wont need to worry about this.
Yeah that's true, I guess this is probably the best argument for doing it this way. Ignore my last email, I'll go ahead and rework stuff to do the grouping at the end. Thanks,
Josef
PATCH 3/5 trace-cmd: lookup syscall names in profile
In reply to Josef BacikWhen we profile syscall times we're just given the syscall number, not the name. This isn't helpful for human beings, so lookup the syscall names when we are printing out the information. This uses the audit library to get this information, I'm not married to it, but it seems that any app that does this mapping has to scrape /usr/include/asm/unistd*.h to pull out the names, so might as well just use somebody elses work.
--- Makefile | 2 +- trace-profile.c | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-)
Makefile +1 -1
@@ -160,7 +160,7 @@ bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) plugin_dir_SQ = $(subst ','\'',$(plugin_dir)) python_dir_SQ = $(subst ','\'',$(python_dir)) -LIBS = -L. -ltracecmd -ldl +LIBS = -L. -ltracecmd -ldl -laudit LIB_FILE = libtracecmd.a PACKAGES= gtk+-2.0 libxml-2.0 gthread-2.0
trace-profile.c +16 -3
@@ -23,6 +23,7 @@#include <stdio.h> #include <stdlib.h> #include <string.h> +#include <libaudit.h> #include "trace-local.h" #include "trace-hash.h"
@@ -916,8 +917,20 @@ static void func_print(struct trace_seq *s, struct event_hash *event_hash)trace_seq_printf (s,"func: 0x%llx" , event_hash->val); } -static void print_int (struct trace_seq *s,struct event_hash *event_hash) +static void syscall_print (struct trace_seq *s,struct event_hash *event_hash) { +const char *name =NULL ; +int machine; + + machine =audit_detect_machine (); +if (machine <0 ) + goto fail; + name =audit_syscall_to_name (event_hash->val, machine); +if (!name) + goto fail; +trace_seq_printf (s,"syscall:%s" , name); +return ; +fail:trace_seq_printf (s,"%s:%d" , event_hash->event_data->event->name, (int )event_hash->val); }
@@ -1425,8 +1438,8 @@ void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hook,if (syscall_enter && syscall_exit) {mate_events (h, syscall_enter,NULL ,"id" , syscall_exit,"id" ,1 ,0 ); - syscall_enter->print_func = print_int; - syscall_exit->print_func = print_int; + syscall_enter->print_func = syscall_print; + syscall_exit->print_func = syscall_print; } events =pevent_list_events (pevent,EVENT_SORT_ID );
Signature · 1 line
2.1.0
Re: [PATCH 3/5] trace-cmd: lookup syscall names in profile
Quoted 18 lines · On Fri, 8 May 2015 15:29:02 -0400 Josef Bacik <[email protected]> wrote:
When we profile syscall times we're just given the syscall number, not the name. This isn't helpful for human beings, so lookup the syscall names when we are printing out the information. This uses the audit library to get this information, I'm not married to it, but it seems that any app that does this mapping has to scrape
/usr/include/asm/unistd*.hto pull out the names, so might as well just use somebody elses work.Signed-off-by Josef Bacik <[email protected]>--- Makefile | 2 +- trace-profile.c | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-)Makefile +0
-0
Makefile bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) · line 163
Quoted 5 lines
Makefile +1
-1@@ -160,7 +160,7 @@ bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) plugin_dir_SQ = $(subst ','\'',$(plugin_dir)) python_dir_SQ = $(subst ','\'',$(python_dir)) -LIBS = -L. -ltracecmd -ldl +LIBS = -L. -ltracecmd -ldl -laudit
This fails to compile for me because I don't have a audit-libs-devel installed.
I'll keep this as is, but I'm going to add code to check for this library, and go back to the default raw numbers if it doesn't exist.
-- Steve
Re: [PATCH 3/5] trace-cmd: lookup syscall names in profile
Quoted 24 lines · Steven Rostedt
Quoted 21 lines · On Fri, 8 May 2015 15:29:02 -0400 Josef Bacik <[email protected]> wrote:
When we profile syscall times we're just given the syscall number, not the name. This isn't helpful for human beings, so lookup the syscall names when we are printing out the information. This uses the audit library to get this information, I'm not married to it, but it seems that any app that does this mapping has to scrape
/usr/include/asm/unistd*.hto pull out the names, so might as well just use somebody elses work.Signed-off-by Josef Bacik <[email protected]>--- Makefile | 2 +- trace-profile.c | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-)Makefile +0
-0@@ -160,7 +160,7 @@ bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) plugin_dir_SQ = $(subst ','\'',$(plugin_dir)) python_dir_SQ = $(subst ','\'',$(python_dir))
Quoted 9 lines
Makefile bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) · line 163
Quoted 2 lines
Makefile +1
-1-LIBS = -L. -ltracecmd -ldl +LIBS = -L. -ltracecmd -ldl -lauditThis fails to compile for me because I don't have a audit-libs-devel installed.
I'll keep this as is, but I'm going to add code to check for this library, and go back to the default raw numbers if it doesn't exist.
Maybe we should think about autoconf at some point? Thanks,
Josef
Re: [PATCH 3/5] trace-cmd: lookup syscall names in profile
Quoted 5 lines · On Wed, 20 May 2015 14:22:31 -0400 Josef Bacik <[email protected]> wrote:
Quoted 3 lines
I'll keep this as is, but I'm going to add code to check for this library, and go back to the default raw numbers if it doesn't exist.
Maybe we should think about autoconf at some point? Thanks,
Egad no! I hate autoconf. And so do many others!
A simple test works fine. I don't want to require libaudit. Just warn users that compile without it.
I like to install trace-cmd on minimal setups (like embedded devices). autoconf will kill that.
-- Steve
Re: [PATCH 3/5] trace-cmd: lookup syscall names in profile
Quoted 9 lines · On Wed, 20 May 2015 14:56:17 -0400 Steven Rostedt <[email protected]> wrote:
Quoted 5 lines · On Wed, 20 May 2015 14:22:31 -0400 Josef Bacik <[email protected]> wrote:
Quoted 3 lines
I'll keep this as is, but I'm going to add code to check for this library, and go back to the default raw numbers if it doesn't exist.
Maybe we should think about autoconf at some point? Thanks,
Quoted 8 lines
Egad no! I hate autoconf. And so do many others!
A simple test works fine. I don't want to require libaudit. Just warn users that compile without it.
I like to install trace-cmd on minimal setups (like embedded devices). autoconf will kill that.
Here's what I did to fix it:
-- Steve
Makefile +16 -3
@@ -160,7 +160,7 @@ bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) plugin_dir_SQ = $(subst ','\'',$(plugin_dir)) python_dir_SQ = $(subst ','\'',$(python_dir)) -LIBS = -L. -ltracecmd -ldl -laudit +LIBS = -L. -ltracecmd -ldl LIB_FILE = libtracecmd.a PACKAGES= gtk+-2.0 libxml-2.0 gthread-2.0
@@ -231,12 +231,25 @@ override CFLAGS += -D_GNU_SOURCE ifndef NO_PTRACE ifneq ($(call try-cc,$(SOURCE_PTRACE),),y) NO_PTRACE = 1 - CFLAGS += -DWARN_NO_PTRACE + override CFLAGS += -DWARN_NO_PTRACE endif endif ifdef NO_PTRACE -CFLAGS += -DNO_PTRACE +override CFLAGS += -DNO_PTRACE +endif + +ifndef NO_AUDIT +ifneq ($(call try-cc,$(SOURCE_AUDIT),-laudit),y) + NO_AUDIT = 1 + override CFLAGS += -DWARN_NO_AUDIT +endif +endif + +ifdef NO_AUDIT +override CFLAGS += -DNO_AUDIT +else +LIBS += -laudit endif # Append required CFLAGS
features.mk +18 -0
@@ -34,3 +34,21 @@ int main (void) return ret; } endef + +define SOURCE_AUDIT +#include <stdio.h> +#include <libaudit.h> + +int main (void) +{ + char *name; + int ret; + ret = audit_detect_machine(); + if (ret < 0) + return ret; + name = audit_syscall_to_name(1, ret); + if (!name) + return -1; + return ret; +} +endef
trace-profile.c +9 -1
@@ -23,11 +23,17 @@#include <stdio.h> #include <stdlib.h> #include <string.h> +#ifndef NO_AUDIT #include <libaudit.h> - +#endif #include "trace-local.h" #include "trace-hash.h" +#ifdef WARN_NO_AUDIT +# warning "lib audit not found, using raw syscalls " \ + "(install libaudit-devel and try again)" +#endif +#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP"#define TASK_STATE_MAX 1024
@@ -848,6 +854,7 @@ static void func_print(struct trace_seq *s, struct event_hash *event_hash)static void syscall_print (struct trace_seq *s,struct event_hash *event_hash) { +#ifndef NO_AUDIT const char *name =NULL ;int machine;
@@ -860,6 +867,7 @@ static void syscall_print(struct trace_seq *s, struct event_hash *event_hash)trace_seq_printf (s,"syscall:%s" , name);return ; fail: +#endif trace_seq_printf (s,"%s:%d" , event_hash->event_data->event->name, (int )event_hash->val); }
Re: [PATCH 3/5] trace-cmd: lookup syscall names in profile
Quoted 125 lines · Steven Rostedt
Quoted 9 lines · On Wed, 20 May 2015 14:56:17 -0400 Steven Rostedt <[email protected]> wrote:
Quoted 5 lines · On Wed, 20 May 2015 14:22:31 -0400 Josef Bacik <[email protected]> wrote:
Quoted 3 lines
I'll keep this as is, but I'm going to add code to check for this library, and go back to the default raw numbers if it doesn't exist.
Maybe we should think about autoconf at some point? Thanks,
Quoted 8 lines
Egad no! I hate autoconf. And so do many others!
A simple test works fine. I don't want to require libaudit. Just warn users that compile without it.
I like to install trace-cmd on minimal setups (like embedded devices). autoconf will kill that.
Here's what I did to fix it:
-- Steve
Makefile +16
-3@@ -160,7 +160,7 @@ bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) plugin_dir_SQ = $(subst ','\'',$(plugin_dir)) python_dir_SQ = $(subst ','\'',$(python_dir)) -LIBS = -L. -ltracecmd -ldl -laudit +LIBS = -L. -ltracecmd -ldl LIB_FILE = libtracecmd.a PACKAGES= gtk+-2.0 libxml-2.0 gthread-2.0@@ -231,12 +231,25 @@ override CFLAGS += -D_GNU_SOURCE ifndef NO_PTRACE ifneq ($(call try-cc,$(SOURCE_PTRACE),),y) NO_PTRACE = 1 - CFLAGS += -DWARN_NO_PTRACE + override CFLAGS += -DWARN_NO_PTRACE endif endif ifdef NO_PTRACE -CFLAGS += -DNO_PTRACE +override CFLAGS += -DNO_PTRACE +endif + +ifndef NO_AUDIT +ifneq ($(call try-cc,$(SOURCE_AUDIT),-laudit),y) + NO_AUDIT = 1 + override CFLAGS += -DWARN_NO_AUDIT +endif +endif + +ifdef NO_AUDIT +override CFLAGS += -DNO_AUDIT +else +LIBS += -laudit endif # Append required CFLAGSfeatures.mk +18
-0@@ -34,3 +34,21 @@ int main (void) return ret; } endef + +define SOURCE_AUDIT +#include <stdio.h> +#include <libaudit.h> + +int main (void) +{ + char *name; + int ret; + ret = audit_detect_machine(); + if (ret < 0) + return ret; + name = audit_syscall_to_name(1, ret); + if (!name) + return -1; + return ret; +} +endeftrace-profile.c +8
-1@@ -23,11 +23,17 @@#include <stdio.h> #include <stdlib.h> #include <string.h> +#ifndef NO_AUDIT #include <libaudit.h> - +#endif #include "trace-local.h" #include "trace-hash.h" +#ifdef WARN_NO_AUDIT +# warning "lib audit not found, using raw syscalls " \ + "(install libaudit-devel and try again)" +#endif +#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP"#define TASK_STATE_MAX 1024@@ -848,6 +854,7 @@ static void func_print(struct trace_seq *s, struct event_hash *event_hash)static void syscall_print (struct trace_seq *s,struct event_hash *event_hash) { +#ifndef NO_AUDIT const char *name =NULL ;int machine;
trace-profile.c static void syscall_print(struct trace_seq *s, struct event_hash *event_hash) · line 873
Quoted 7 lines
trace-profile.c +1
-0@@ -860,6 +867,7 @@ static void syscall_print(struct trace_seq *s, struct event_hash *event_hash)trace_seq_printf (s,"syscall:%s" , name);return ; fail: +#endif trace_seq_printf (s,"%s:%d" , event_hash->event_data->event->name, (int )event_hash->val); }
Looks good to me, thanks,
Josef
PATCH 4/5 trace-cmd: sort the events in the profile output
In reply to Josef BacikCurrently we just spit out the events as they appear in the hash, which makes it hard to tell where we are spending our time. Instead order the output by type and total time spent. So you'll get something like this
syscall 1: 100000ms syscall 2: 80000ms syscall 3: 4ms func 1: 10ms func 2: 1ms
instead of them all jumbled up.
--- trace-profile.c | 4 ++++ 1 file changed, 4 insertions(+)
trace-profile.c +4 -0
@@ -1961,6 +1961,10 @@ static int compare_events(const void *a, const void *b)return 1 ;if (event_data_a->id < event_data_b->id)return -1 ; +if ((*A )->time_total > (*B )->time_total) +return -1 ; +if ((*A )->time_total < (*B )->time_total) +return 1 ;return 0 ; }
Signature · 1 line
2.1.0
PATCH 5/5 trace-cmd: make trace-hist work with instances
In reply to Josef BacikIf you recorded information completely in an instance then trace-hist wouldn't think your trace.dat file had any events because it didn't know about instances. Fix this.
--- trace-hist.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-)
trace-hist.c +25 -8
@@ -954,13 +954,6 @@ static void do_trace_hist(struct tracecmd_input *handle)int cpu;int ret; - ret =tracecmd_init_data (handle); -if (ret <0 ) -die ("failed to init data" ); - -if (ret >0 ) -die ("trace-cmd hist does not work with latency traces\n" ); - cpus =tracecmd_cpus (handle);/* Need to get any event */
@@ -1025,6 +1018,7 @@ void trace_hist(int argc, char **argv) {struct tracecmd_input *handle;const char *input_file =NULL ; +int instances;int ret;for (;;) {
@@ -1067,7 +1061,30 @@ void trace_hist(int argc, char **argv)if (ret)return ; -do_trace_hist (handle); + ret =tracecmd_init_data (handle); +if (ret <0 ) +die ("failed to init data" ); + +if (ret >0 ) +die ("trace-cmd hist does not work with latency traces\n" ); + + instances =tracecmd_buffer_instances (handle); +if (instances) { +struct tracecmd_input *new_handle; +int i; + +for (i =0 ; i < instances; i++) { + new_handle =tracecmd_buffer_instance_handle (handle, i); +if (!new_handle) { +warning ("could not retrieve handle %d" , i); +continue ; + } +do_trace_hist (new_handle); +tracecmd_close (new_handle); + } + }else { +do_trace_hist (handle); + }tracecmd_close (handle); }
Signature · 1 line
2.1.0
Quoted 14 lines · On Fri, 8 May 2015 15:28:59 -0400 Josef Bacik <[email protected]> wrote:
I had to use trace-cmd profile to debug a regression with hhvm recently. These are the patches I needed to make trace-cmd profile useful to me. There are a few changes in here
1) Adding --merge to trace-cmd profile. This merges events for like comms together. HHVM is a huge app that spawns worker threads to handle requests, I needed all of this data in one place to see how the app as a whole was behaving, --merge provides this.
2) Fixup the output formatting. I also wanted to see which syscalls were taking the longest but this was hard because we were just spitting out the syscall number and it wasn't in any order, so now all the events are ordered based on time taken and the syscall names are printed instead of the number.
Quoted 5 lines
3) Random trace-cmd hist fix, because it didn't work right with instances.
I've been using these in production for a couple of weeks now and they seem to work well. Thanks,
Josef,
Thanks a lot! I really appreciate this.
I'll take a look at it next week as I have other things on my plate for today. Ping me if I somehow forget to do so.
Thanks again,
-- Steve