perf_counter: dynamically allocate tasks' perf_counter_context struct
Subsystems (4)
RFC PATCH 1/1 perf_counter: dynamically allocate tasks' perf_counter_context struct
This replaces the struct perf_counter_context in the task_struct with a pointer to a dynamically allocated perf_counter_context struct. The main reason for doing is this is to allow us to transfer a perf_counter_context from one task to another when we do lazy PMU switching in a later patch.
This has a few side-benefits: the task_struct becomes a little smaller, we save some memory because only tasks that have perf_counters attached get a perf_counter_context allocated for them, and we can remove the inclusion of <linux/perf_counter.h> in sched.h, meaning that we don't end up recompiling nearly everything whenever perf_counter.h changes.
As well as the pointer to the perf_counter_context in the task_struct, this adds a mutex to provide mutual exclusion between writers of the pointer.
The perf_counter_context structures are reference-counted and freed when the last reference is dropped. A context can have references from its task, counters and the perf_cpu_context for the cpu where the task is running. Counters can outlive the task so it is possible that a context will be freed well after its task has exited.
This removes the task pointer from the perf_counter struct. The task pointer was not used anywhere and would make it harder to move a context from one task to another. Anything that needed to know which task a counter was attached to was already using counter->ctx->task.
The __perf_counter_init_context function moves up in perf_counter.c so that it can be called from find_get_context, and now initializes the refcount, but is otherwise unchanged.
We were potentially calling list_del_counter twice: once from __perf_counter_exit_task when the task exits and once from __perf_counter_remove_from_context when the counter's fd gets closed. This adds a check in list_del_counter so it doesn't do anything if the counter has already been removed from the lists.
Since perf_counter_task_sched_in doesn't do anything if the task doesn't have a context, and leaves cpuctx->task_ctx = NULL, this adds code to __perf_install_in_context to set cpuctx->task_ctx if necessary, i.e. in the case where the current task adds the first counter to itself and thus creates a context for itself.
This also adds similar code to __perf_counter_enable to handle a similar situation which can arise when the counters have been disabled using prctl; that also leaves cpuctx->task_ctx = NULL.
--- arch/x86/kernel/apic/apic.c | 1 + include/linux/init_task.h | 8 +-- include/linux/perf_counter.h | 5 +- include/linux/sched.h | 7 +- kernel/exit.c | 3 +- kernel/fork.c | 1 + kernel/perf_counter.c | 180 ++++++++++++++++++++++++++++++------------ 7 files changed, 141 insertions(+), 64 deletions(-)
arch/x86/kernel/apic/apic.c +1 -0
@@ -14,6 +14,7 @@* Mikael Pettersson : PM converted to driver model. */ +#include <linux/perf_counter.h> #include <linux/kernel_stat.h> #include <linux/mc146818rtc.h> #include <linux/acpi_pmtmr.h>
include/linux/init_task.h +2 -6
@@ -110,12 +110,8 @@ extern struct cred init_cred;#ifdef CONFIG_PERF_COUNTERS # define INIT_PERF_COUNTERS (tsk) \ - .perf_counter_ctx.counter_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.counter_list), \ - .perf_counter_ctx.event_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.event_list), \ - .perf_counter_ctx.lock = \ - __SPIN_LOCK_UNLOCKED(tsk.perf_counter_ctx.lock), + .perf_counter_mutex = \ + __MUTEX_INITIALIZER(tsk.perf_counter_mutex),#else # define INIT_PERF_COUNTERS (tsk)#endif
include/linux/perf_counter.h +2 -3
@@ -441,7 +441,6 @@ struct perf_counter {struct hw_perf_counter hw;struct perf_counter_context *ctx; -struct task_struct *task;struct file *filp;struct perf_counter *parent;
@@ -490,7 +489,6 @@ struct perf_counter {* Used as a container for task counters and CPU counters as well: */ struct perf_counter_context { -#ifdef CONFIG_PERF_COUNTERS /* * Protect the states of the counters in the list, * nr_active, and the list:
@@ -516,7 +514,8 @@ struct perf_counter_context {*/ u64 time;u64 timestamp; -#endif + +atomic_t refcount; };/**
include/linux/sched.h +5 -2
@@ -71,7 +71,6 @@ struct sched_param {#include <linux/path.h> #include <linux/compiler.h> #include <linux/completion.h> -#include <linux/perf_counter.h> #include <linux/pid.h> #include <linux/percpu.h> #include <linux/topology.h>
@@ -99,6 +98,7 @@ struct robust_list_head;struct bio ;struct bts_tracer ;struct fs_struct ; +struct perf_counter_context ;/* * List of flags we want to share for kernel threads,
@@ -1387,7 +1387,10 @@ struct task_struct {struct list_head pi_state_list;struct futex_pi_state *pi_state_cache;#endif -struct perf_counter_context perf_counter_ctx; +#ifdef CONFIG_PERF_COUNTERS +struct perf_counter_context *perf_counter_ctx; +struct mutex perf_counter_mutex; +#endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy;short il_next;
kernel/exit.c +2 -1
@@ -48,6 +48,7 @@#include <linux/tracehook.h> #include <linux/fs_struct.h> #include <linux/init_task.h> +#include <linux/perf_counter.h> #include <trace/sched.h> #include <asm/uaccess.h>
@@ -159,7 +160,7 @@ static void delayed_put_task_struct(struct rcu_head *rhp) struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);#ifdef CONFIG_PERF_COUNTERS -WARN_ON_ONCE (!list_empty (&tsk->perf_counter_ctx.counter_list)); +WARN_ON_ONCE (tsk->perf_counter_ctx);#endif trace_sched_process_free (tsk);put_task_struct (tsk);
kernel/fork.c +1 -0
@@ -63,6 +63,7 @@#include <linux/fs_struct.h> #include <trace/sched.h> #include <linux/magic.h> +#include <linux/perf_counter.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
kernel/perf_counter.c +128 -52
403 lines not on this page · raw message
Signature · 1 line
1.6.0.4
Quoted 5 lines · Paul Mackerras
This replaces the struct
perf_counter_contextin thetask_structwith a pointer to a dynamically allocatedperf_counter_contextstruct. The main reason for doing is this is to allow us to transfer aperf_counter_contextfrom one task to another when we do lazy PMU switching in a later patch.
Looks good!, few comments below.
Quoted 7 lines
(quoted diff) +7
-0+ ctx = task->perf_counter_ctx; + if (!ctx) { + ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL); + if (!ctx) { + put_task_struct(task); + return ERR_PTR(-ENOMEM); + }
Review
Quoted 10 lines
(quoted diff) +10
-0+ __perf_counter_init_context(ctx, task); + mutex_lock(&task->perf_counter_mutex); + if (!task->perf_counter_ctx) { + task->perf_counter_ctx = ctx; + } else { + kfree(ctx); + ctx = task->perf_counter_ctx; + } + mutex_unlock(&task->perf_counter_mutex); + }
This seems to be the only site where we use ->perf_counter_mutex, couldn't we simply write this as:
if (cmpxchg(&task->perf_counter_ctx, NULL, ctx) != NULL)
kfree(ctx);
and get rid of the mutex?
Quoted 9 lines
(quoted diff) +4
-0+/* + * Remove a counter from the lists for its context. + * Must be called with counter->mutex and ctx->mutex held. + */ static void list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
Review __perf_counter_exit_task(struct task_struct *child, · line 3361
Quoted 8 lines
(quoted diff) +2
-0@@ -3295,6 +3354,8 @@ __perf_counter_exit_task(struct task_struct *child, * * Note: we may be running in child context, but the PID is not hashed * anymore so new counters will not be added. + * (XXX not sure that is true when we get called from flush_old_exec. + * -- paulus) */ void perf_counter_exit_task(struct task_struct *child) {
Yes, we have a little hole here, but I think it should fixable.
we have:
do_exit()
{
exit_signals(); /* sets PF_EXITING */
... (1)
perf_counter_exit_task();
... (2)
<unhash somewhere>
vs
find_get_context()
If, in find_get_context() we exclude PF_EXITING tasks under ctx->mutex, and ensure we take ctx->mutex in perf_counter_exit_task() we should be good.
Tasks could race and still attach in (1), but that would be ok, as they'd get cleared out in perf_counter_exit_task(), but not after that in (2) as they'd be sure to observe PF_EXITING.
Now it appears perf_counter_exit_task() is a little light on locking, so the below patch adds both ctx->mutex and counter->mutex, as you say it should when invoking list_del_counter().
Secondly it adds the PF_EXITING test in find_get_context().
(utterly untested and such..)
---
kernel/perf_counter.c +28 -30
@@ -1228,6 +1228,13 @@ static struct perf_counter_context *find ctx = &task->perf_counter_ctx; ctx->task = task; +mutex_lock (&ctx->mutex); +if (task->state &PF_EXITING ) { +mutex_unlock (&ctx->mutex); +return ERR_PTR (-ESRCH ); + } +mutex_unlock (&ctx->mutex); +/* Reuse ptrace permission checks for now. */ if (!ptrace_may_access (task,PTRACE_MODE_READ )) {put_context (ctx);
@@ -3292,42 +3299,29 @@ __perf_counter_exit_task(struct task_str struct perf_counter_context *child_ctx) {struct perf_counter *parent_counter; +struct perf_cpu_context *cpuctx; +unsigned long flags;/* -* If we do not self-reap then we have to wait for the -* child task to unschedule (it will happen for sure), -* so that its counter is at its final count. (This -* condition triggers rarely - child tasks usually get -* off their CPU before the parent has a chance to -* get this far into the reaping action) -*/ -if (child != current) { -wait_task_inactive (child,0 ); -update_counter_times (child_counter); -list_del_counter (child_counter, child_ctx); - }else { -struct perf_cpu_context *cpuctx; -unsigned long flags; - -/* -* Disable and unlink this counter. -* -* Be careful about zapping the list - IRQ/NMI context -* could still be processing it: -*/ -local_irq_save (flags); -perf_disable (); +* Disable and unlink this counter. +* +* Be careful about zapping the list - IRQ/NMI context +* could still be processing it: +*/ +local_irq_save (flags); +perf_disable (); - cpuctx = &__get_cpu_var (perf_cpu_context); + cpuctx = &__get_cpu_var (perf_cpu_context); -group_sched_out (child_counter, cpuctx, child_ctx); -update_counter_times (child_counter); +group_sched_out (child_counter, cpuctx, child_ctx); +update_counter_times (child_counter); -list_del_counter (child_counter, child_ctx); +perf_enable (); +local_irq_restore (flags); -perf_enable (); -local_irq_restore (flags); - } +mutex_lock (&child_counter->mutex); +list_del_counter (child_counter, child_ctx); +mutex_unlock (&child_counter->mutex); parent_counter = child_counter->parent;/*
@@ -3359,6 +3353,8 @@ void perf_counter_exit_task(struct task_if (likely (!child_ctx->nr_counters))return ; +mutex_lock (&child_ctx->mutex); + again:list_for_each_entry_safe (child_counter, tmp, &child_ctx->counter_list, list_entry)
@@ -3371,6 +3367,8 @@ again:*/ if (!list_empty (&child_ctx->counter_list)) goto again; + +mutex_unlock (&child_ctx->mutex); }/*
Quoted 7 lines · Peter Zijlstra
This seems to be the only site where we use ->perf_counter_mutex, couldn't we simply write this as:
if (
cmpxchg(&task->perf_counter_ctx, NULL, ctx)!= NULL)
kfree(ctx);and get rid of the mutex?
Yes, that should work, with an smp_wmb() before the cmpxchg.
Quoted 3 lines
If, in
find_get_context()we excludePF_EXITINGtasks underctx->mutex, and ensure we takectx->mutexinperf_counter_exit_task()we should be good.
It's the fact that we also call perf_counter_exit_task when doing an exec of a setuid program that concerns me. In that case the task isn't exiting and its pid is still accessible, so it is quite possible that some other task could be attaching counters to us. Certainly root could; I don't know whether we change uid before doing the dumpable check or not, so it's possible a non-privileged process would be refused.
Paul.
Quoted 5 lines · Paul Mackerras
This replaces the struct
perf_counter_contextin thetask_structwith a pointer to a dynamically allocatedperf_counter_contextstruct. The main reason for doing is this is to allow us to transfer aperf_counter_contextfrom one task to another when we do lazy PMU switching in a later patch.
Hm, i'm not sure how far this gets us towards lazy PMU switching.
In fact i'd say that the term "lazy PMU switching" is probably misleading, we should use: "equivalent PMU context switching" or instead.
The difference is really crucial. We cannot really detach a PMU context from a task, because the task might migrate to another CPU and could run it there. Any lazyness in the switching of the PMU context would create the need to send IPIs and other overhead. For similar reasons are lazy FPU switching methods not workable on SMP generally.
Instead, the right abstraction is to define 'equivalency' between task's PMU contexts, created by inheritance. When two tasks context-switch that both have the same parent counter(s), we dont need to do _any_ physical PMU switching. The counts (and events) from one of the tasks can be freely transferred to the other task. It's going to get summarized in the parent anyway, so context-switching is an invariant.
To implement this, we need something like an 'ID', cookie or generation counter for the context, which changes to another unique number (or pointer) the moment a context is modified: a counter is added, removed or a counter attribute is changed. When counters are inherited the cookie gets carried over too. The context-switch code can then do this optimization:
if (prev->ctx.cookie != next->ctx.cookie) switch_pmu_ctx(prev, next);
... which will be _very_ fast for the inherited counters (perf stat) case.
Note, this does put a few requirements on the architecture code, and it requires a few changes to the sched-in/sched-out code and requires a few changes to when tasks migrate to other CPUs.
For example the x86 code currently demuxes counter events back to counter pointers, using a per-cpu structure:
struct cpu_hw_counters {
struct perf_counter *counters[X86_PMC_IDX_MAX];
unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
unsigned long interrupts;
int enabled;};
the counter pointers are per task - so this bit of cpu_hw_counters needs to move into the ctx structure, so that if an overflow IRQ comes in, we always only deal with local counters (not with some previous task's counter pointers).
Ingo
Quoted 8 lines · Ingo Molnar
Quoted 5 lines · Paul Mackerras
This replaces the struct
perf_counter_contextin thetask_structwith a pointer to a dynamically allocatedperf_counter_contextstruct. The main reason for doing is this is to allow us to transfer aperf_counter_contextfrom one task to another when we do lazy PMU switching in a later patch.
Quoted 5 lines
Hm, i'm not sure how far this gets us towards lazy PMU switching.
In fact i'd say that the term "lazy PMU switching" is probably misleading, we should use: "equivalent PMU context switching" or instead.
Yes, that's what I mean.
As you say, we need to be able to detect when two tasks have equivalent contexts - that is, when their counters are all inherited from a common ancestor. My idea is that in that situation we simply swap the contexts: move the context of the outgoing task onto the incoming task, and give the incoming task's context to the outgoing task. With my patch, that involves simply swapping the pointers over and adjusting the task pointers in the two contexts.
That means that all the counters get transferred over to the incoming task, so there is nothing in the PMU or the arch code that needs to changed or adjusted. The outgoing task still has a perfectly valid context, so it doesn't matter if it migrates to another CPU. The nice thing is that there is nothing special or unusual about the state after we have swapped the contexts - nothing that needs to be remembered or undone later.
Hopefully I'll have an actual patch to show you later today.
Paul.
Quoted 27 lines · Paul Mackerras
Quoted 8 lines · Ingo Molnar
Quoted 5 lines · Paul Mackerras
This replaces the struct
perf_counter_contextin thetask_structwith a pointer to a dynamically allocatedperf_counter_contextstruct. The main reason for doing is this is to allow us to transfer aperf_counter_contextfrom one task to another when we do lazy PMU switching in a later patch.Quoted 5 lines
Hm, i'm not sure how far this gets us towards lazy PMU switching.
In fact i'd say that the term "lazy PMU switching" is probably misleading, we should use: "equivalent PMU context switching" or instead.
Yes, that's what I mean.
As you say, we need to be able to detect when two tasks have equivalent contexts - that is, when their counters are all inherited from a common ancestor. My idea is that in that situation we simply swap the contexts: move the context of the outgoing task onto the incoming task, and give the incoming task's context to the outgoing task. With my patch, that involves simply swapping the pointers over and adjusting the task pointers in the two contexts.
Quoted 7 lines
That means that all the counters get transferred over to the incoming task, so there is nothing in the PMU or the arch code that needs to changed or adjusted. The outgoing task still has a perfectly valid context, so it doesn't matter if it migrates to another CPU. The nice thing is that there is nothing special or unusual about the state after we have swapped the contexts - nothing that needs to be remembered or undone later.
Yeah, agreed - your scheme is simpler than the scheme i thought of!
Ingo