perf counter bits
PATCH 4/4 perf_counter: optimize sched in/out of counters
In reply to Peter ZijlstraAvoid a function call for !group counters by directly calling the counter function.
--- kernel/perf_counter.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
kernel/perf_counter.c +19 -6
@@ -826,8 +826,12 @@ void __perf_counter_sched_out(struct perperf_disable ();if (ctx->nr_active) { -list_for_each_entry (counter, &ctx->counter_list, list_entry) -group_sched_out (counter, cpuctx, ctx); +list_for_each_entry (counter, &ctx->counter_list, list_entry) { +if (counter != counter->group_leader) +counter_sched_out (counter, cpuctx, ctx); +else +group_sched_out (counter, cpuctx, ctx); + } }perf_enable (); out:
@@ -903,8 +907,12 @@ __perf_counter_sched_in(struct perf_counif (counter->cpu !=-1 && counter->cpu != cpu)continue ; -if (group_can_go_on (counter, cpuctx,1 )) -group_sched_in (counter, cpuctx, ctx, cpu); +if (counter != counter->group_leader) +counter_sched_in (counter, cpuctx, ctx, cpu); +else { +if (group_can_go_on (counter, cpuctx,1 )) +group_sched_in (counter, cpuctx, ctx, cpu); + }/* * If this pinned group hasn't been scheduled,
@@ -932,9 +940,14 @@ __perf_counter_sched_in(struct perf_counif (counter->cpu !=-1 && counter->cpu != cpu)continue ; -if (group_can_go_on (counter, cpuctx, can_add_hw)) { -if (group_sched_in (counter, cpuctx, ctx, cpu)) +if (counter != counter->group_leader) { +if (counter_sched_in (counter, cpuctx, ctx, cpu)) can_add_hw =0 ; + }else { +if (group_can_go_on (counter, cpuctx, can_add_hw)) { +if (group_sched_in (counter, cpuctx, ctx, cpu)) + can_add_hw =0 ; + } } }perf_enable ();
PATCH 3/4 perf_counter: optimize disable of time based sw counters
In reply to Peter ZijlstraCurrently we call hrtimer_cancel() unconditionally on disable of time based software counters. Avoid when possible.
--- kernel/perf_counter.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
kernel/perf_counter.c +4 -2
@@ -2716,7 +2716,8 @@ static int cpu_clock_perf_counter_enablestatic void cpu_clock_perf_counter_disable (struct perf_counter *counter) { -hrtimer_cancel (&counter->hw.hrtimer); +if (counter->hw.irq_period) +hrtimer_cancel (&counter->hw.hrtimer);cpu_clock_perf_counter_update (counter); }
@@ -2767,7 +2768,8 @@ static int task_clock_perf_counter_enablstatic void task_clock_perf_counter_disable (struct perf_counter *counter) { -hrtimer_cancel (&counter->hw.hrtimer); +if (counter->hw.irq_period) +hrtimer_cancel (&counter->hw.hrtimer);task_clock_perf_counter_update (counter, counter->ctx->time); }
cover perf counter bits
PATCH 2/4 perf_counter: log irq_period changes
For the dynamic irq_period code, log whenever we change the period so that analyzing code can normalize the event flow.
--- include/linux/perf_counter.h | 8 ++++++++ kernel/perf_counter.c | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-)
include/linux/perf_counter.h +8 -0
@@ -258,6 +258,14 @@ enum perf_event_type {PERF_EVENT_COMM =3 ,/* +* struct { +* struct perf_event_header header; +* u64 irq_period; +* }; +*/ + PERF_EVENT_PERIOD = 4, + +/* * When header.misc & PERF_EVENT_MISC_OVERFLOW the event_type field * will be PERF_RECORD_* *
kernel/perf_counter.c +39 -1
@@ -1046,7 +1046,9 @@ int perf_counter_task_enable(void)return 0 ; } -void perf_adjust_freq (struct perf_counter_context *ctx) +static void perf_log_period (struct perf_counter *counter,u64 period); + +static void perf_adjust_freq (struct perf_counter_context *ctx) {struct perf_counter *counter;u64 irq_period;
@@ -1072,6 +1074,8 @@ void perf_adjust_freq(struct perf_counteif (!irq_period) irq_period =1 ; +perf_log_period (counter, irq_period); + counter->hw.irq_period = irq_period; counter->hw.interrupts =0 ; }
@@ -2407,6 +2411,40 @@ void perf_counter_munmap(unsigned long a }/* +* +*/ + +static void perf_log_period (struct perf_counter *counter,u64 period) +{ +struct perf_output_handle handle; +int ret; + +struct { +struct perf_event_header header; +u64 time; +u64 period; + } freq_event = { + .header = { + .type =PERF_EVENT_PERIOD , + .misc =0 , + .size =sizeof (freq_event), + }, + .time =sched_clock (), + .period = period, + }; + +if (counter->hw.irq_period == period) +return ; + + ret =perf_output_begin (&handle, counter,sizeof (freq_event),0 ,0 ); +if (ret) +return ; + +perf_output_put (&handle, freq_event); +perf_output_end (&handle); +} + +/* * Generic counter overflow handling. */
PATCH 1/4 perf_counter: solve the rotate_ctx vs inherit race differently
In reply to Peter ZijlstraInstead of disabling RR scheduling of the counters, use a different list that does not get rotated to iterate the counters on inheritance.
LKML-Reference: <new-submission>
--- include/linux/perf_counter.h | 1 - kernel/perf_counter.c | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-)
include/linux/perf_counter.h +0 -1
@@ -508,7 +508,6 @@ struct perf_counter_context {int nr_counters;int nr_active;int is_active; -int rr_allowed;struct task_struct *task;/*
kernel/perf_counter.c +5 -10
@@ -1120,8 +1120,7 @@ void perf_counter_task_tick(struct task___perf_counter_task_sched_out (ctx);rotate_ctx (&cpuctx->ctx); -if (ctx->rr_allowed) -rotate_ctx (ctx); +rotate_ctx (ctx);perf_counter_cpu_sched_in (cpuctx, cpu);perf_counter_task_sched_in (curr, cpu);
@@ -3109,7 +3108,6 @@ __perf_counter_init_context(struct perf_mutex_init (&ctx->mutex);INIT_LIST_HEAD (&ctx->counter_list);INIT_LIST_HEAD (&ctx->event_list); - ctx->rr_allowed =1 ; ctx->task = task; }
@@ -3350,14 +3348,14 @@ void perf_counter_init_task(struct task_*/ mutex_lock (&parent_ctx->mutex); - parent_ctx->rr_allowed =0 ; -barrier ();/* irqs */ -/* * We dont have to disable NMIs - we are only looking at * the list, not manipulating it: */ -list_for_each_entry (counter, &parent_ctx->counter_list, list_entry) { +list_for_each_entry_rcu (counter, &parent_ctx->event_list, event_entry) { +if (counter != counter->group_leader) +continue ; +if (!counter->hw_event.inherit)continue ;
@@ -3366,9 +3364,6 @@ void perf_counter_init_task(struct task_break ; } -barrier ();/* irqs */ - parent_ctx->rr_allowed =1 ; -mutex_unlock (&parent_ctx->mutex); }
Ingo Molnar
AppliedApplied to tip:perfcounters/core, thanks Peter!
Ingo
[tip:perfcounters/core] perf_counter: Solve the rotate_ctx vs inherit race differently
In reply to Peter ZijlstraCommit-ID: d7b629a34fc4134a43c730b5f0197855dc4948d0 Gitweb: http://git.kernel.org/tip/d7b629a34fc4134a43c730b5f0197855dc4948d0 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 20 May 2009 12:21:19 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 20 May 2009 12:43:32 +0200
perf_counter: Solve the rotate_ctx vs inherit race differently
Instead of disabling RR scheduling of the counters, use a different list that does not get rotated to iterate the counters on inheritance.
[ Impact: cleanup, optimization ]
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 1 - kernel/perf_counter.c | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-)
include/linux/perf_counter.h +0 -1
@@ -508,7 +508,6 @@ struct perf_counter_context {int nr_counters;int nr_active;int is_active; -int rr_allowed;struct task_struct *task;/*
kernel/perf_counter.c +5 -10
@@ -1120,8 +1120,7 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)__perf_counter_task_sched_out (ctx);rotate_ctx (&cpuctx->ctx); -if (ctx->rr_allowed) -rotate_ctx (ctx); +rotate_ctx (ctx);perf_counter_cpu_sched_in (cpuctx, cpu);perf_counter_task_sched_in (curr, cpu);
@@ -3109,7 +3108,6 @@ __perf_counter_init_context(struct perf_counter_context *ctx,mutex_init (&ctx->mutex);INIT_LIST_HEAD (&ctx->counter_list);INIT_LIST_HEAD (&ctx->event_list); - ctx->rr_allowed =1 ; ctx->task = task; }
@@ -3350,14 +3348,14 @@ void perf_counter_init_task(struct task_struct *child)*/ mutex_lock (&parent_ctx->mutex); - parent_ctx->rr_allowed =0 ; -barrier ();/* irqs */ -/* * We dont have to disable NMIs - we are only looking at * the list, not manipulating it: */ -list_for_each_entry (counter, &parent_ctx->counter_list, list_entry) { +list_for_each_entry_rcu (counter, &parent_ctx->event_list, event_entry) { +if (counter != counter->group_leader) +continue ; +if (!counter->hw_event.inherit)continue ;
@@ -3366,9 +3364,6 @@ void perf_counter_init_task(struct task_struct *child)break ; } -barrier ();/* irqs */ - parent_ctx->rr_allowed =1 ; -mutex_unlock (&parent_ctx->mutex); }
[tip:perfcounters/core] perf_counter: Optimize disable of time based sw counters
In reply to Peter ZijlstraCommit-ID: b986d7ec0f8b7ea3cc7366d80a137fbe839df227 Gitweb: http://git.kernel.org/tip/b986d7ec0f8b7ea3cc7366d80a137fbe839df227 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 20 May 2009 12:21:21 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 20 May 2009 12:43:33 +0200
perf_counter: Optimize disable of time based sw counters
Currently we call hrtimer_cancel() unconditionally on disable of time based software counters. Avoid when possible.
[ Impact: micro-optimize the code ]
LKML-Reference: <[email protected]>
--- kernel/perf_counter.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
kernel/perf_counter.c +4 -2
@@ -2716,7 +2716,8 @@ static int cpu_clock_perf_counter_enable(struct perf_counter *counter)static void cpu_clock_perf_counter_disable (struct perf_counter *counter) { -hrtimer_cancel (&counter->hw.hrtimer); +if (counter->hw.irq_period) +hrtimer_cancel (&counter->hw.hrtimer);cpu_clock_perf_counter_update (counter); }
@@ -2767,7 +2768,8 @@ static int task_clock_perf_counter_enable(struct perf_counter *counter)static void task_clock_perf_counter_disable (struct perf_counter *counter) { -hrtimer_cancel (&counter->hw.hrtimer); +if (counter->hw.irq_period) +hrtimer_cancel (&counter->hw.hrtimer);task_clock_perf_counter_update (counter, counter->ctx->time); }
[tip:perfcounters/core] perf_counter: Log irq_period changes
In reply to Peter ZijlstraCommit-ID: 26b119bc811a73bac6ecf95bdf284bf31c7955f0 Gitweb: http://git.kernel.org/tip/26b119bc811a73bac6ecf95bdf284bf31c7955f0 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 20 May 2009 12:21:20 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 20 May 2009 12:43:33 +0200
perf_counter: Log irq_period changes
For the dynamic irq_period code, log whenever we change the period so that analyzing code can normalize the event flow.
[ Impact: add new feature to allow more precise profiling ]
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 8 ++++++++ kernel/perf_counter.c | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletions(-)
include/linux/perf_counter.h +8 -0
@@ -258,6 +258,14 @@ enum perf_event_type {PERF_EVENT_COMM =3 ,/* +* struct { +* struct perf_event_header header; +* u64 irq_period; +* }; +*/ + PERF_EVENT_PERIOD = 4, + +/* * When header.misc & PERF_EVENT_MISC_OVERFLOW the event_type field * will be PERF_RECORD_* *
kernel/perf_counter.c +39 -1
@@ -1046,7 +1046,9 @@ int perf_counter_task_enable(void)return 0 ; } -void perf_adjust_freq (struct perf_counter_context *ctx) +static void perf_log_period (struct perf_counter *counter,u64 period); + +static void perf_adjust_freq (struct perf_counter_context *ctx) {struct perf_counter *counter;u64 irq_period;
@@ -1072,6 +1074,8 @@ void perf_adjust_freq(struct perf_counter_context *ctx)if (!irq_period) irq_period =1 ; +perf_log_period (counter, irq_period); + counter->hw.irq_period = irq_period; counter->hw.interrupts =0 ; }
@@ -2407,6 +2411,40 @@ void perf_counter_munmap(unsigned long addr, unsigned long len, }/* +* +*/ + +static void perf_log_period (struct perf_counter *counter,u64 period) +{ +struct perf_output_handle handle; +int ret; + +struct { +struct perf_event_header header; +u64 time; +u64 period; + } freq_event = { + .header = { + .type =PERF_EVENT_PERIOD , + .misc =0 , + .size =sizeof (freq_event), + }, + .time =sched_clock (), + .period = period, + }; + +if (counter->hw.irq_period == period) +return ; + + ret =perf_output_begin (&handle, counter,sizeof (freq_event),0 ,0 ); +if (ret) +return ; + +perf_output_put (&handle, freq_event); +perf_output_end (&handle); +} + +/* * Generic counter overflow handling. */
[tip:perfcounters/core] perf_counter: Optimize sched in/out of counters
In reply to Peter ZijlstraCommit-ID: afedadf23a2c90f3ba0d963282cbe6a6be129494 Gitweb: http://git.kernel.org/tip/afedadf23a2c90f3ba0d963282cbe6a6be129494 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 20 May 2009 12:21:22 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 20 May 2009 12:43:34 +0200
perf_counter: Optimize sched in/out of counters
Avoid a function call for !group counters by directly calling the counter function.
[ Impact: micro-optimize the code ]
LKML-Reference: <[email protected]>
--- kernel/perf_counter.c | 25 +++++++++++++++++++------ 1 files changed, 19 insertions(+), 6 deletions(-)
kernel/perf_counter.c +19 -6
@@ -826,8 +826,12 @@ void __perf_counter_sched_out(struct perf_counter_context *ctx,perf_disable ();if (ctx->nr_active) { -list_for_each_entry (counter, &ctx->counter_list, list_entry) -group_sched_out (counter, cpuctx, ctx); +list_for_each_entry (counter, &ctx->counter_list, list_entry) { +if (counter != counter->group_leader) +counter_sched_out (counter, cpuctx, ctx); +else +group_sched_out (counter, cpuctx, ctx); + } }perf_enable (); out:
@@ -903,8 +907,12 @@ __perf_counter_sched_in(struct perf_counter_context *ctx,if (counter->cpu !=-1 && counter->cpu != cpu)continue ; -if (group_can_go_on (counter, cpuctx,1 )) -group_sched_in (counter, cpuctx, ctx, cpu); +if (counter != counter->group_leader) +counter_sched_in (counter, cpuctx, ctx, cpu); +else { +if (group_can_go_on (counter, cpuctx,1 )) +group_sched_in (counter, cpuctx, ctx, cpu); + }/* * If this pinned group hasn't been scheduled,
@@ -932,9 +940,14 @@ __perf_counter_sched_in(struct perf_counter_context *ctx,if (counter->cpu !=-1 && counter->cpu != cpu)continue ; -if (group_can_go_on (counter, cpuctx, can_add_hw)) { -if (group_sched_in (counter, cpuctx, ctx, cpu)) +if (counter != counter->group_leader) { +if (counter_sched_in (counter, cpuctx, ctx, cpu)) can_add_hw =0 ; + }else { +if (group_can_go_on (counter, cpuctx, can_add_hw)) { +if (group_sched_in (counter, cpuctx, ctx, cpu)) + can_add_hw =0 ; + } } }perf_enable ();