perf_counter: dynamically allocate tasks' perf_counter_context struct [v2]
Subsystems (4)
PATCH 1/2 perf_counter: dynamically allocate tasks' perf_counter_context struct [v2]
This replaces the struct perf_counter_context in the task_struct with a pointer to a dynamically allocated perf_counter_context struct. The main reason for doing is this is to allow us to transfer a perf_counter_context from one task to another when we do lazy PMU switching in a later patch.
This has a few side-benefits: the task_struct becomes a little smaller, we save some memory because only tasks that have perf_counters attached get a perf_counter_context allocated for them, and we can remove the inclusion of <linux/perf_counter.h> in sched.h, meaning that we don't end up recompiling nearly everything whenever perf_counter.h changes.
The perf_counter_context structures are reference-counted and freed when the last reference is dropped. A context can have references from its task and the counters on its task. Counters can outlive the task so it is possible that a context will be freed well after its task has exited.
Contexts are allocated on fork if the parent had a context, or otherwise the first time that a per-task counter is created on a task. In the latter case, we set the context pointer in the task struct locklessly using an atomic compare-and-exchange operation in case we raced with some other task in creating a context for the subject task.
This also removes the task pointer from the perf_counter struct. The task pointer was not used anywhere and would make it harder to move a context from one task to another. Anything that needed to know which task a counter was attached to was already using counter->ctx->task.
The __perf_counter_init_context function moves up in perf_counter.c so that it can be called from find_get_context, and now initializes the refcount, but is otherwise unchanged.
We were potentially calling list_del_counter twice: once from __perf_counter_exit_task when the task exits and once from __perf_counter_remove_from_context when the counter's fd gets closed. This adds a check in list_del_counter so it doesn't do anything if the counter has already been removed from the lists.
Since perf_counter_task_sched_in doesn't do anything if the task doesn't have a context, and leaves cpuctx->task_ctx = NULL, this adds code to __perf_install_in_context to set cpuctx->task_ctx if necessary, i.e. in the case where the current task adds the first counter to itself and thus creates a context for itself.
This also adds similar code to __perf_counter_enable to handle a similar situation which can arise when the counters have been disabled using prctl; that also leaves cpuctx->task_ctx = NULL.
--- arch/x86/kernel/apic/apic.c | 1 + include/linux/init_task.h | 13 --- include/linux/perf_counter.h | 4 +- include/linux/sched.h | 6 +- kernel/exit.c | 3 +- kernel/fork.c | 1 + kernel/perf_counter.c | 218 ++++++++++++++++++++++++++---------------- 7 files changed, 145 insertions(+), 101 deletions(-)
arch/x86/kernel/apic/apic.c +1 -0
@@ -14,6 +14,7 @@* Mikael Pettersson : PM converted to driver model. */ +#include <linux/perf_counter.h> #include <linux/kernel_stat.h> #include <linux/mc146818rtc.h> #include <linux/acpi_pmtmr.h>
include/linux/init_task.h +0 -13
@@ -108,18 +108,6 @@ extern struct group_info init_groups;extern struct cred init_cred; -#ifdef CONFIG_PERF_COUNTERS -# define INIT_PERF_COUNTERS (tsk) \ - .perf_counter_ctx.counter_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.counter_list), \ - .perf_counter_ctx.event_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.event_list), \ - .perf_counter_ctx.lock = \ - __SPIN_LOCK_UNLOCKED(tsk.perf_counter_ctx.lock), -#else -# define INIT_PERF_COUNTERS (tsk) -#endif -/* * INIT_TASK is used to set up the first task table, touch at * your own risk!. Base=0, limit=0x1fffff (=2MB)
@@ -183,7 +171,6 @@ extern struct cred init_cred; }, \ .dirties = INIT_PROP_LOCAL_SINGLE(dirties), \INIT_IDS \ -INIT_PERF_COUNTERS (tsk) \ INIT_TRACE_IRQFLAGS \INIT_LOCKDEP \INIT_FTRACE_GRAPH \
include/linux/perf_counter.h +1 -3
@@ -449,7 +449,6 @@ struct perf_counter {struct hw_perf_counter hw;struct perf_counter_context *ctx; -struct task_struct *task;struct file *filp;struct perf_counter *parent;
@@ -498,7 +497,6 @@ struct perf_counter {* Used as a container for task counters and CPU counters as well: */ struct perf_counter_context { -#ifdef CONFIG_PERF_COUNTERS /* * Protect the states of the counters in the list, * nr_active, and the list:
@@ -516,6 +514,7 @@ struct perf_counter_context {int nr_counters;int nr_active;int is_active; +atomic_t refcount;struct task_struct *task;/*
@@ -523,7 +522,6 @@ struct perf_counter_context {*/ u64 time;u64 timestamp; -#endif };/**
include/linux/sched.h +4 -2
@@ -71,7 +71,6 @@ struct sched_param {#include <linux/path.h> #include <linux/compiler.h> #include <linux/completion.h> -#include <linux/perf_counter.h> #include <linux/pid.h> #include <linux/percpu.h> #include <linux/topology.h>
@@ -99,6 +98,7 @@ struct robust_list_head;struct bio ;struct bts_tracer ;struct fs_struct ; +struct perf_counter_context ;/* * List of flags we want to share for kernel threads,
@@ -1387,7 +1387,9 @@ struct task_struct {struct list_head pi_state_list;struct futex_pi_state *pi_state_cache;#endif -struct perf_counter_context perf_counter_ctx; +#ifdef CONFIG_PERF_COUNTERS +struct perf_counter_context *perf_counter_ctxp; +#endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy;short il_next;
kernel/exit.c +2 -1
@@ -48,6 +48,7 @@#include <linux/tracehook.h> #include <linux/fs_struct.h> #include <linux/init_task.h> +#include <linux/perf_counter.h> #include <trace/sched.h> #include <asm/uaccess.h>
@@ -159,7 +160,7 @@ static void delayed_put_task_struct(struct rcu_head *rhp) struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);#ifdef CONFIG_PERF_COUNTERS -WARN_ON_ONCE (!list_empty (&tsk->perf_counter_ctx.counter_list)); +WARN_ON_ONCE (tsk->perf_counter_ctxp);#endif trace_sched_process_free (tsk);put_task_struct (tsk);
kernel/fork.c +1 -0
@@ -63,6 +63,7 @@#include <linux/fs_struct.h> #include <trace/sched.h> #include <linux/magic.h> +#include <linux/perf_counter.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
kernel/perf_counter.c +136 -82
440 lines not on this page · raw message
Signature · 1 line
1.6.0.4
PATCH 2/2 perf_counter: optimize context switch between identical inherited contexts
When monitoring a process and its descendants with a set of inherited counters, we can often get the situation in a context switch where both the old (outgoing) and new (incoming) process have the same set of counters, and their values are ultimately going to be added together. In that situation it doesn't matter which set of counters are used to count the activity for the new process, so there is really no need to go through the process of reading the hardware counters and updating the old task's counters and then setting up the PMU for the new task.
This optimizes the context switch in this situation. Instead of scheduling out the perf_counter_context for the old task and scheduling in the new context, we simply transfer the old context to the new task and keep using it without interruption. The new context gets transferred to the old task. This means that both tasks still have a valid perf_counter_context, so no special case is introduced when the old task gets scheduled in again, either on this CPU or another CPU.
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Here are some measurements of the context switch time as measured with the lat_ctx benchmark from lmbench, comparing the times obtained with and without this patch series:
-----Unmodified----- With this patch series
Counters: none 2 HW 4H+4S none 2 HW 4H+4S
2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60
St dev 0.04 0.04 0.13 0.05 0.17 0.19
8 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57
St dev 1.27 1.04 0.88 1.42 1.46 1.42
32 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15
St dev 0.41 0.47 0.53 0.54 0.57 0.81
The numbers are the mean and standard deviation of 20 runs of lat_ctx. The "none" columns are lat_ctx run directly without any counters. The "2 HW" columns are with lat_ctx run under perfstat, counting cycles and instructions. The "4H+4S" columns are lat_ctx run under perfstat with 4 hardware counters and 4 software counters (cycles, instructions, cache references, cache misses, task clock, context switch, cpu migrations, and page faults).
--- include/linux/perf_counter.h | 12 ++++- kernel/perf_counter.c | 109 ++++++++++++++++++++++++++++++++++++----- kernel/sched.c | 2 +- 3 files changed, 107 insertions(+), 16 deletions(-)
include/linux/perf_counter.h +11 -1
@@ -513,6 +513,7 @@ struct perf_counter_context {struct list_head event_list;int nr_counters;int nr_active; +int nr_enabled;int is_active;atomic_t refcount;struct task_struct *task;
@@ -522,6 +523,14 @@ struct perf_counter_context {*/ u64 time;u64 timestamp; + +/* +* These fields let us detect when two contexts have both +* been cloned (inherited) from a common ancestor. +*/ +struct perf_counter_context *parent_ctx; +u32 parent_gen; +u32 generation; };/**
@@ -552,7 +561,8 @@ extern int perf_max_counters;extern const struct pmu *hw_perf_counter_init (struct perf_counter *counter);extern void perf_counter_task_sched_in (struct task_struct *task,int cpu); -extern void perf_counter_task_sched_out (struct task_struct *task,int cpu); +extern void perf_counter_task_sched_out (struct task_struct *task, +struct task_struct *next,int cpu);extern void perf_counter_task_tick (struct task_struct *task,int cpu);extern void perf_counter_init_task (struct task_struct *child);extern void perf_counter_exit_task (struct task_struct *child);
kernel/perf_counter.c +95 -14
219 lines not on this page · raw message
kernel/sched.c +1 -1
@@ -5091,7 +5091,7 @@ need_resched_nonpreemptible:if (likely (prev != next)) {sched_info_switch (prev, next); -perf_counter_task_sched_out (prev, cpu); +perf_counter_task_sched_out (prev, next, cpu); rq->nr_switches++; rq->curr = next;
Signature · 1 line
1.6.0.4
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 6 lines · Paul Mackerras
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Curious point that.. so prctl() can disable counters it doesn't own.
Shouldn't we instead fix that?
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Paul Mackerras
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Curious point that.. so
prctl()can disable counters it doesn't own.Shouldn't we instead fix that?
Well, prctl enables/disables the counters that are counting on the current process, regardless of who or what created them. I always thought that was a little strange; maybe it is useful to be able to disable all the counters that any other process might have put on to you, but I can't think of a scenario where you'd really need to do that, particularly since the disable is a one-shot operation, and doesn't prevent new (enabled) counters being attached to you.
On the other hand, what does "all the counters I own" mean? Does it mean all the ones that I have fds open for? Or does it mean all the ones that I created? Either way we don't have a good way to enumerate them.
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 22 lines · Paul Mackerras
Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Paul Mackerras
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Curious point that.. so
prctl()can disable counters it doesn't own.Shouldn't we instead fix that?
Well, prctl enables/disables the counters that are counting on the current process, regardless of who or what created them. I always thought that was a little strange; maybe it is useful to be able to disable all the counters that any other process might have put on to you, but I can't think of a scenario where you'd really need to do that, particularly since the disable is a one-shot operation, and doesn't prevent new (enabled) counters being attached to you.
Quoted 4 lines
On the other hand, what does "all the counters I own" mean? Does it mean all the ones that I have fds open for? Or does it mean all the ones that I created? Either way we don't have a good way to enumerate them.
I'm for all counters you created (ie have a fd for). Being able to disable counters others created on you just sounds wrong.
If we can settle on a semantic, I'm sure we can implement it :-)
Ingo, Corey, any opinions?
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 28 lines · Peter Zijlstra
Quoted 22 lines · Paul Mackerras
Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Paul Mackerras
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Curious point that.. so
prctl()can disable counters it doesn't own.Shouldn't we instead fix that?
Well, prctl enables/disables the counters that are counting on the current process, regardless of who or what created them. I always thought that was a little strange; maybe it is useful to be able to disable all the counters that any other process might have put on to you, but I can't think of a scenario where you'd really need to do that, particularly since the disable is a one-shot operation, and doesn't prevent new (enabled) counters being attached to you.
Quoted 4 lines
On the other hand, what does "all the counters I own" mean? Does it mean all the ones that I have fds open for? Or does it mean all the ones that I created? Either way we don't have a good way to enumerate them.
Quoted 6 lines
I'm for all counters you created (ie have a fd for). Being able to disable counters others created on you just sounds wrong.
If we can settle on a semantic, I'm sure we can implement it :-)
Ingo, Corey, any opinions?
It indeed doesnt sound correct that we can disable counters others created on us - especially if they are in a different (higher privileged) security context than us.
OTOH, enabling/disabling counters in specific functions of a library might be a valid use-case. So perhaps make this an attribute:
.transparent or so, with perf stat defaulting on it to be transparent (i.e. not child context disable-able).
Ingo
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 7 lines · Ingo Molnar
Quoted 6 lines
I'm for all counters you created (ie have a fd for). Being able to disable counters others created on you just sounds wrong.
If we can settle on a semantic, I'm sure we can implement it :-)
Ingo, Corey, any opinions?
Quoted 8 lines
It indeed doesnt sound correct that we can disable counters others created on us - especially if they are in a different (higher privileged) security context than us.
OTOH, enabling/disabling counters in specific functions of a library might be a valid use-case. So perhaps make this an attribute:
..transparent or so, with perf stat defaulting on it to be transparent (i.e. not child context disable-able).
I'm not sure that's something we want to do. Furthermore, if we do want it, the current implementation is not sufficient, because, as Paul noted, we can attach a new counter right after the disable.
I really think such limitations should come from whatever security policy there is on attaching counters. Eg. using selinux, label gnupg as non-countable so that you simply cannot attach (incl inherit) counters to it.
Allowing such hooks into libraries will destroy transparency for developers and I don't think that's something we'd want to promote.
I'll implement my suggestion so we can take it from there.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Ingo MolnarQuoted 2 lines · Ingo Molnar
OTOH, enabling/disabling counters in specific functions of a library might be a valid use-case.
If that's the use case, wouldn't we wouldn't the "enable" to restore the previous state, rather than enabling all counters?
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Paul Mackerrasinclude/linux/perf_counter.h struct perf_counter_context { · line 533
Quoted 13 lines · Paul Mackerras
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
include/linux/perf_counter.h +1
-0+u32 generation;
Suppose someone writes a malicious proglet that inherits the counters, puts the child to sleep, does 2^32 mods on the counter set, and then wakes up the child.
Would that merely corrupt the results, or make the kernel explode?
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 14 lines · Peter Zijlstra
Quoted 11 lines · Paul Mackerras
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
Quoted 7 lines
include/linux/perf_counter.h struct perf_counter_context { · line 533
Quoted 1 line
include/linux/perf_counter.h +1
-0+u32 generation;Suppose someone writes a malicious proglet that inherits the counters, puts the child to sleep, does 2^32 mods on the counter set, and then wakes up the child.
Would that merely corrupt the results, or make the kernel explode?
i think it should be an u64.
Ingo
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 22 lines · Ingo Molnar
Quoted 20 lines · Peter Zijlstra
include/linux/perf_counter.h struct perf_counter_context { · line 533
Quoted 13 lines · Paul Mackerras
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
include/linux/perf_counter.h +1
-0+u32 generation;Suppose someone writes a malicious proglet that inherits the counters, puts the child to sleep, does 2^32 mods on the counter set, and then wakes up the child.
Quoted 3 lines
Quoted 1 line
Would that merely corrupt the results, or make the kernel explode?
i think it should be an u64.
So at to make the attack infeasible by death of the attacker and burnout of the solar-system? :-)
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 23 lines · Peter Zijlstra
Quoted 22 lines · Ingo Molnar
Quoted 20 lines · Peter Zijlstra
include/linux/perf_counter.h struct perf_counter_context { · line 533
Quoted 13 lines · Paul Mackerras
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
include/linux/perf_counter.h +1
-0+u32 generation;Suppose someone writes a malicious proglet that inherits the counters, puts the child to sleep, does 2^32 mods on the counter set, and then wakes up the child.
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Would that merely corrupt the results, or make the kernel explode?
i think it should be an u64.
So at to make the attack infeasible by death of the attacker and burnout of the solar-system? :-)
Yeah. We only have to make it for the next 1 billion years anyway - then due to the sun being ~50% brighter the planet as we know it will be scorched up anyway. (Unless some entity puts it under massive space based shades until then, to preserve the ancient biosphere for nostalgic or museologic reasons or so.)
Ingo
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
Suppose someone writes a malicious proglet that inherits the counters, puts the child to sleep, does 2^32 mods on the counter set, and then wakes up the child.
Would that merely corrupt the results, or make the kernel explode?
You'd have to do something like create some counters on yourself, fork, do the 2^32 counter creations and deletions, then do another fork. All that would happen is that some of the counters would count on one of the child processes when they should be counting on the other.
But it would be easy and cheap to make the generation count be 64 bits, and then it won't overflow in my lifetime at least, and after that I don't care. :) So I agree with Ingo that we should just do that.
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Paul MackerrasQuoted 8 lines · Paul Mackerras
-----Unmodified----- With this patch series
Counters: none 2 HW 4H+4S none 2 HW 4H+4S2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60 St dev 0.04 0.04 0.13 0.05 0.17 0.19
Quoted 7 lines
8 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57 St dev 1.27 1.04 0.88 1.42 1.46 1.4232 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15 St dev 0.41 0.47 0.53 0.54 0.57 0.81
Any clues as to why the time is still dependent on the number of counters in the context? The approach seems to be O(1) in that it does a simple counter context swap on sched_out and nothing on sched_in.
Peter Zijlstra
maintainerRe: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Paul Mackerras
-----Unmodified----- With this patch series
Counters: none 2 HW 4H+4S none 2 HW 4H+4S2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60 St dev 0.04 0.04 0.13 0.05 0.17 0.198 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57 St dev 1.27 1.04 0.88 1.42 1.46 1.42
Quoted 7 lines
Quoted 3 lines
32 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15 St dev 0.41 0.47 0.53 0.54 0.57 0.81Any clues as to why the time is still dependent on the number of counters in the context? The approach seems to be O(1) in that it does a simple counter context swap on
sched_outand nothing onsched_in.
Ah, should I read this as 'n' lmbench instances on a single cpu? So that we get multiple inheritance sets mixed and have to switch between them?
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 2 lines · Peter Zijlstra
Ah, should I read this as 'n' lmbench instances on a single cpu? So that we get multiple inheritance sets mixed and have to switch between them?
No, it's one lat_ctx instance, which creates N processes connected in a ring with pipes, and passes a token around some number of times and measures the overall time to do that. It subtracts off the time it takes to do all the pipe reads and writes in a single process to get an estimate of how much the context switches are costing.
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Peter ZijlstraQuoted 13 lines · Peter Zijlstra
Quoted 12 lines · Paul Mackerras
-----Unmodified----- With this patch series
Counters: none 2 HW 4H+4S none 2 HW 4H+4S2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60 St dev 0.04 0.04 0.13 0.05 0.17 0.198 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57 St dev 1.27 1.04 0.88 1.42 1.46 1.42
Quoted 7 lines
Quoted 3 lines
32 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15 St dev 0.41 0.47 0.53 0.54 0.57 0.81Any clues as to why the time is still dependent on the number of counters in the context? The approach seems to be O(1) in that it does a simple counter context swap on
sched_outand nothing onsched_in.
Only the context switches between the lat_ctx processes will be optimized; switches between them and other processes will still do the full PMU switch. The CPU would be switching to other processes from time to time during the run (e.g. to run various kernel daemons, which seem to continue to proliferate) so some fraction of the context switches would be as expensive as before. Probably I should measure what that fraction is, but it's Friday night and I'm feeling lazy. :)
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Paul MackerrasQuoted 65 lines · Paul Mackerras
When monitoring a process and its descendants with a set of inherited counters, we can often get the situation in a context switch where both the old (outgoing) and new (incoming) process have the same set of counters, and their values are ultimately going to be added together. In that situation it doesn't matter which set of counters are used to count the activity for the new process, so there is really no need to go through the process of reading the hardware counters and updating the old task's counters and then setting up the PMU for the new task.
This optimizes the context switch in this situation. Instead of scheduling out the
perf_counter_contextfor the old task and scheduling in the new context, we simply transfer the old context to the new task and keep using it without interruption. The new context gets transferred to the old task. This means that both tasks still have a validperf_counter_context, so no special case is introduced when the old task gets scheduled in again, either on this CPU or another CPU.The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Here are some measurements of the context switch time as measured with the
lat_ctxbenchmark from lmbench, comparing the times obtained with and without this patch series:-----Unmodified----- With this patch seriesCounters: none 2 HW 4H+4S none 2 HW 4H+4S
2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60
St dev 0.04 0.04 0.13 0.05 0.17 0.198 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57
St dev 1.27 1.04 0.88 1.42 1.46 1.4232 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15
St dev 0.41 0.47 0.53 0.54 0.57 0.81The numbers are the mean and standard deviation of 20 runs of
lat_ctx. The "none" columns arelat_ctxrun directly without any counters. The "2 HW" columns are withlat_ctxrun under perfstat, counting cycles and instructions. The "4H+4S" columns arelat_ctxrun under perfstat with 4 hardware counters and 4 software counters (cycles, instructions, cache references, cache misses, task clock, context switch, cpu migrations, and page faults).
Quoted 6 lines
Signed-off-by Paul Mackerras <[email protected]>--- include/linux/perf_counter.h | 12 ++++- kernel/perf_counter.c | 109 ++++++++++++++++++++++++++++++++++++----- kernel/sched.c | 2 +- 3 files changed, 107 insertions(+), 16 deletions(-)
Impressive!
I'm wondering where the sensitivity of lat_ctx on the number of counters comes from. I'd expect there to be constant (and very low) overhead. It could be measurement noise - lat_ctx is very sensitive on L2 layout and memory allocation patterns - those are very hard to eliminate and are not measured via the stddev numbers. (many of those effects are per bootup specific, and bootups dont randomize them - so there's no easy way to measure their statistical impact.)
Ingo
[tip:perfcounters/core] perf_counter: Optimize context switch between identical inherited contexts
In reply to Paul MackerrasCommit-ID: 564c2b210add41df9a3a5aaa365c1d97cff6110d Gitweb: http://git.kernel.org/tip/564c2b210add41df9a3a5aaa365c1d97cff6110d Author: Paul Mackerras <[email protected]> AuthorDate: Fri, 22 May 2009 14:27:22 +1000 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 22 May 2009 12:18:20 +0200
perf_counter: Optimize context switch between identical inherited contexts
When monitoring a process and its descendants with a set of inherited counters, we can often get the situation in a context switch where both the old (outgoing) and new (incoming) process have the same set of counters, and their values are ultimately going to be added together. In that situation it doesn't matter which set of counters are used to count the activity for the new process, so there is really no need to go through the process of reading the hardware counters and updating the old task's counters and then setting up the PMU for the new task.
This optimizes the context switch in this situation. Instead of scheduling out the perf_counter_context for the old task and scheduling in the new context, we simply transfer the old context to the new task and keep using it without interruption. The new context gets transferred to the old task. This means that both tasks still have a valid perf_counter_context, so no special case is introduced when the old task gets scheduled in again, either on this CPU or another CPU.
The equivalence of contexts is detected by keeping a pointer in each cloned context pointing to the context it was cloned from. To cope with the situation where a context is changed by adding or removing counters after it has been cloned, we also keep a generation number on each context which is incremented every time a context is changed. When a context is cloned we take a copy of the parent's generation number, and two cloned contexts are equivalent only if they have the same parent and the same generation number. In order that the parent context pointer remains valid (and is not reused), we increment the parent context's reference count for each context cloned from it.
Since we don't have individual fds for the counters in a cloned context, the only thing that can make two clones of a given parent different after they have been cloned is enabling or disabling all counters with prctl. To account for this, we keep a count of the number of enabled counters in each context. Two contexts must have the same number of enabled counters to be considered equivalent.
Here are some measurements of the context switch time as measured with the lat_ctx benchmark from lmbench, comparing the times obtained with and without this patch series:
-----Unmodified----- With this patch series
Counters: none 2 HW 4H+4S none 2 HW 4H+4S
2 processes:
Average 3.44 6.45 11.24 3.12 3.39 3.60
St dev 0.04 0.04 0.13 0.05 0.17 0.19
8 processes:
Average 6.45 8.79 14.00 5.57 6.23 7.57
St dev 1.27 1.04 0.88 1.42 1.46 1.42
32 processes:
Average 5.56 8.43 13.78 5.28 5.55 7.15
St dev 0.41 0.47 0.53 0.54 0.57 0.81
The numbers are the mean and standard deviation of 20 runs of lat_ctx. The "none" columns are lat_ctx run directly without any counters. The "2 HW" columns are with lat_ctx run under perfstat, counting cycles and instructions. The "4H+4S" columns are lat_ctx run under perfstat with 4 hardware counters and 4 software counters (cycles, instructions, cache references, cache misses, task clock, context switch, cpu migrations, and page faults).
[ Impact: performance optimization of counter context-switches ]
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 12 ++++- kernel/perf_counter.c | 109 ++++++++++++++++++++++++++++++++++++----- kernel/sched.c | 2 +- 3 files changed, 107 insertions(+), 16 deletions(-)
include/linux/perf_counter.h +11 -1
@@ -513,6 +513,7 @@ struct perf_counter_context {struct list_head event_list;int nr_counters;int nr_active; +int nr_enabled;int is_active;atomic_t refcount;struct task_struct *task;
@@ -522,6 +523,14 @@ struct perf_counter_context {*/ u64 time;u64 timestamp; + +/* +* These fields let us detect when two contexts have both +* been cloned (inherited) from a common ancestor. +*/ +struct perf_counter_context *parent_ctx; +u32 parent_gen; +u32 generation; };/**
@@ -552,7 +561,8 @@ extern int perf_max_counters;extern const struct pmu *hw_perf_counter_init (struct perf_counter *counter);extern void perf_counter_task_sched_in (struct task_struct *task,int cpu); -extern void perf_counter_task_sched_out (struct task_struct *task,int cpu); +extern void perf_counter_task_sched_out (struct task_struct *task, +struct task_struct *next,int cpu);extern void perf_counter_task_tick (struct task_struct *task,int cpu);extern void perf_counter_init_task (struct task_struct *child);extern void perf_counter_exit_task (struct task_struct *child);
kernel/perf_counter.c +95 -14
219 lines not on this page · raw message
kernel/sched.c +1 -1
@@ -5091,7 +5091,7 @@ need_resched_nonpreemptible:if (likely (prev != next)) {sched_info_switch (prev, next); -perf_counter_task_sched_out (prev, cpu); +perf_counter_task_sched_out (prev, next, cpu); rq->nr_switches++; rq->curr = next;
Re: [tip:perfcounters/core] perf_counter: Optimize context switch between identical inherited contexts
kernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task, int cpu) · line 945
Quoted 13 lines · tip-bot for Paul Mackerras
kernel/perf_counter.c +9
-0@@ -885,6 +934,16 @@ void perf_counter_task_sched_out(struct task_struct *task, int cpu) regs =task_pt_regs (task);perf_swcounter_event (PERF_COUNT_CONTEXT_SWITCHES ,1 ,1 , regs,0 ); + + next_ctx = next->perf_counter_ctxp; +if (next_ctx &&context_equiv (ctx, next_ctx)) { + task->perf_counter_ctxp = next_ctx; + next->perf_counter_ctxp = ctx; + ctx->task = next; + next_ctx->task = task; +return ; + }
there's one complication that this trick is causing - the migration counter relies on ctx->task to get per task migration stats:
static inline u64 get_cpu_migrations(struct perf_counter *counter)
{
struct task_struct *curr = counter->ctx->task;
if (curr)
return curr->se.nr_migrations;
return cpu_nr_migrations(smp_processor_id());}
as ctx->task is now jumping (while we keep the context), the migration stats are out of whack.
Ingo
Re: [tip:perfcounters/core] perf_counter: Optimize context switch between identical inherited contexts
Quoted 26 lines · Ingo Molnar
kernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task, int cpu) · line 945
Quoted 13 lines · tip-bot for Paul Mackerras
kernel/perf_counter.c +9
-0@@ -885,6 +934,16 @@ void perf_counter_task_sched_out(struct task_struct *task, int cpu) regs =task_pt_regs (task);perf_swcounter_event (PERF_COUNT_CONTEXT_SWITCHES ,1 ,1 , regs,0 ); + + next_ctx = next->perf_counter_ctxp; +if (next_ctx &&context_equiv (ctx, next_ctx)) { + task->perf_counter_ctxp = next_ctx; + next->perf_counter_ctxp = ctx; + ctx->task = next; + next_ctx->task = task; +return ; + }there's one complication that this trick is causing - the migration counter relies on
ctx->taskto get per task migration stats:static inline u64
get_cpu_migrations(struct perf_counter *counter){ struct task_struct *curr = counter->ctx->task; if (curr) return curr->se.nr_migrations; return cpu_nr_migrations(smp_processor_id());
Quoted 4 lines
}
as
ctx->taskis now jumping (while we keep the context), the migration stats are out of whack.
How did you notice this? The overall sum over all children should still be correct, though some individual children's counters could go negative, so the result of a read on the counter when some children have exited and others haven't could look a bit strange. Reading the counter after all children have exited should be fine, though.
One of the effects of optimizing the context switch is that in general, reading the value of an inheritable counter when some children have exited but some are still running might produce results that include some of the activity of the still-running children and might not include all of the activity of the children that have exited. If that's a concern then we need to implement the "sync child counters" ioctl that has been suggested.
As for the migration counter, it is the only software counter that is still using the "old" approach, i.e. it doesn't generate interrupts and it uses the counter->prev_state field (which I hope to eliminate one day). It's also the only software counter which counts events that happen while the task is not scheduled in. The cleanest thing would be to rewrite the migration counter code to have a callin from the scheduler when migrations happen.
Paul.
Re: [tip:perfcounters/core] perf_counter: Optimize context switch between identical inherited contexts
Quoted 33 lines · Paul Mackerras
Quoted 26 lines · Ingo Molnar
kernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task, int cpu) · line 945
Quoted 13 lines · tip-bot for Paul Mackerras
kernel/perf_counter.c +9
-0@@ -885,6 +934,16 @@ void perf_counter_task_sched_out(struct task_struct *task, int cpu) regs =task_pt_regs (task);perf_swcounter_event (PERF_COUNT_CONTEXT_SWITCHES ,1 ,1 , regs,0 ); + + next_ctx = next->perf_counter_ctxp; +if (next_ctx &&context_equiv (ctx, next_ctx)) { + task->perf_counter_ctxp = next_ctx; + next->perf_counter_ctxp = ctx; + ctx->task = next; + next_ctx->task = task; +return ; + }there's one complication that this trick is causing - the migration counter relies on
ctx->taskto get per task migration stats:static inline u64
get_cpu_migrations(struct perf_counter *counter){ struct task_struct *curr = counter->ctx->task; if (curr) return curr->se.nr_migrations; return cpu_nr_migrations(smp_processor_id());Quoted 4 lines
}
as
ctx->taskis now jumping (while we keep the context), the migration stats are out of whack.
Quoted 6 lines
How did you notice this? The overall sum over all children should still be correct, though some individual children's counters could go negative, so the result of a read on the counter when some children have exited and others haven't could look a bit strange. Reading the counter after all children have exited should be fine, though.
i've noticed a few weirdnesses and then added a debug check and noticed the negative delta values.
Quoted 8 lines
One of the effects of optimizing the context switch is that in general, reading the value of an inheritable counter when some children have exited but some are still running might produce results that include some of the activity of the still-running children and might not include all of the activity of the children that have exited. If that's a concern then we need to implement the "sync child counters" ioctl that has been suggested.
Quoted 7 lines
As for the migration counter, it is the only software counter that is still using the "old" approach, i.e. it doesn't generate interrupts and it uses the
counter->prev_statefield (which I hope to eliminate one day). It's also the only software counter which counts events that happen while the task is not scheduled in. The cleanest thing would be to rewrite the migration counter code to have a callin from the scheduler when migrations happen.
I'll check with the debug check removed again. If the end result is OK then i dont think we need to worry much about this, at this stage.
Ingo
[tip:perfcounters/core] perf_counter: fix !PERF_COUNTERS build failure
In reply to Paul MackerrasCommit-ID: 910431c7f2e963017d767b29c80ae706421e569f Gitweb: http://git.kernel.org/tip/910431c7f2e963017d767b29c80ae706421e569f Author: Ingo Molnar <[email protected]> AuthorDate: Fri, 22 May 2009 12:32:15 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 22 May 2009 12:33:14 +0200
perf_counter: fix !PERF_COUNTERS build failure
Update the !CONFIG_PERF_COUNTERS prototype too, for perf_counter_task_sched_out().
[ Impact: build fix ]
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
include/linux/perf_counter.h +2 -1
@@ -625,7 +625,8 @@ extern void perf_counter_init(void);static inline void perf_counter_task_sched_in (struct task_struct *task,int cpu) { }static inline void -perf_counter_task_sched_out (struct task_struct *task,int cpu) { } +perf_counter_task_sched_out (struct task_struct *task, +struct task_struct *next,int cpu) { }static inline void perf_counter_task_tick (struct task_struct *task,int cpu) { }static inline void perf_counter_init_task (struct task_struct *child) { }
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Paul Mackerraskernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task, int cpu) · line 945
Quoted 8 lines · Paul Mackerras
kernel/perf_counter.c +8
-0+ next_ctx = next->perf_counter_ctxp; +if (next_ctx &&context_equiv (ctx, next_ctx)) { + task->perf_counter_ctxp = next_ctx; + next->perf_counter_ctxp = ctx; + ctx->task = next; + next_ctx->task = task; +return ; + }
Ingo just pointed out that there is nothing there to close the race with attaching a counter.
That is, you could end up attaching your counter to the wrong task.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 4 lines · Peter Zijlstra
Ingo just pointed out that there is nothing there to close the race with attaching a counter.
That is, you could end up attaching your counter to the wrong task.
Good point. Doing the unclone in find_get_context would be a start, but the locking could get tricky (in fact we don't have any way to remove an inherited counter from a context, so we only have to worry about counters being attached). I'll work out a solution after I have digested your recent batch of patches.
Paul.
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 5 lines · Paul Mackerras
Quoted 3 lines · Peter Zijlstra
Ingo just pointed out that there is nothing there to close the race with attaching a counter.
Quoted 7 lines
Quoted 1 line
That is, you could end up attaching your counter to the wrong task.
Good point. Doing the unclone in
find_get_contextwould be a start, but the locking could get tricky (in fact we don't have any way to remove an inherited counter from a context, so we only have to worry about counters being attached). I'll work out a solution after I have digested your recent batch of patches.
I'm currently staring at something like the below, trying to find races etc.. ;-)
attach vs destroy vs flip
---
kernel/perf_counter.c +61 -13
@@ -102,13 +102,29 @@ static void get_ctx(struct perf_counter_atomic_inc (&ctx->refcount); } -static void put_ctx (struct perf_counter_context *ctx) +static void free_ctx_rcu (struct rcu_head *head) +{ +struct perf_counter_context *ctx; + + ctx = container_of(head, struct perf_counter_context, rcu_head); +kfree (ctx); +} + +static bool __put_ctx (struct perf_counter_context *ctx) {if (atomic_dec_and_test (&ctx->refcount)) {if (ctx->parent_ctx)put_ctx (ctx->parent_ctx); -kfree (ctx); +return true; } + +return false; +} + +static void put_ctx (struct perf_counter_context *ctx) +{ +if (__put_ctx (ctx)) +call_rcu (&ctx->rcu_head, free_ctx_rcu); }/*
@@ -934,8 +950,16 @@ void perf_counter_task_sched_out(struct next_ctx = next->perf_counter_ctxp;if (next_ctx &&context_equiv (ctx, next_ctx)) { + ctx->task =NULL ; + next_ctx->task =NULL ; + +smp_wmb (); + task->perf_counter_ctxp = next_ctx; next->perf_counter_ctxp = ctx; + +smp_wmb (); + ctx->task = next; next_ctx->task = task;return ;
@@ -1284,19 +1308,31 @@ static struct perf_counter_context *findreturn ERR_PTR (-EACCES ); } +rcu_read_lock (); +again: ctx = task->perf_counter_ctxp; +/* +* matched against the xchg() in perf_counter_exit_task() setting +* ctx to NULL and the cmpxchg() below. +*/ +smp_read_barrier_depends ();if (!ctx) { +rcu_read_unlock (); +/* +* cannot attach counters to a dying task. +*/ +if (task->flags &PF_EXITING ) { +put_task_struct (task); +return ERR_PTR (-ESRCH ); + } + ctx =kmalloc (sizeof (struct perf_counter_context ),GFP_KERNEL );if (!ctx) {put_task_struct (task);return ERR_PTR (-ENOMEM ); }__perf_counter_init_context (ctx, task); -/* -* Make sure other cpus see correct values for *ctx -* once task->perf_counter_ctxp is visible to them. -*/ -smp_wmb (); +rcu_read_lock (); tctx =cmpxchg (&task->perf_counter_ctxp,NULL , ctx);if (tctx) {/*
@@ -1308,6 +1344,16 @@ static struct perf_counter_context *find } } +if (!atomic_inc_not_zero (&ctx->reference)) + goto again; + +if (rcu_dereference (ctx->task) != task) { +put_ctx (ctx); + goto again; + } + +rcu_read_unlock (); +return ctx; }
@@ -1316,7 +1362,6 @@ static void free_counter_rcu(struct rcu_struct perf_counter *counter; counter = container_of(head, struct perf_counter, rcu_head); -put_ctx (counter->ctx);kfree (counter); }
@@ -1337,6 +1382,7 @@ static void free_counter(struct perf_couif (counter->destroy) counter->destroy (counter); +put_ctx (counter->ctx);call_rcu (&counter->rcu_head, free_counter_rcu); }
@@ -3231,6 +3277,7 @@ SYSCALL_DEFINE5(perf_counter_open, out_fput:fput_light (group_file, fput_needed); +put_ctx (ctx);return ret;
@@ -3390,25 +3437,25 @@ __perf_counter_exit_task(struct task_str* * Note: we may be running in child context, but the PID is not hashed * anymore so new counters will not be added. -* (XXX not sure that is true when we get called from flush_old_exec. -* -- paulus) */ void perf_counter_exit_task (struct task_struct *child) {struct perf_counter *child_counter, *tmp;struct perf_counter_context *child_ctx;unsigned long flags; +bool free_ctx;WARN_ON_ONCE (child != current); - child_ctx = child->perf_counter_ctxp; + child_ctx =xchg (&child->perf_counter_ctxp,NULL );if (likely (!child_ctx))return ; + free_ctx =__put_ctx (child_ctx); +local_irq_save (flags);__perf_counter_task_sched_out (child_ctx); - child->perf_counter_ctxp =NULL ;local_irq_restore (flags);mutex_lock (&child_ctx->mutex);
@@ -3428,7 +3475,8 @@ again:mutex_unlock (&child_ctx->mutex); -put_ctx (child_ctx); +if (free_ctx) +call_rcu (&ctx->rcu_head, free_ctx_rcu); }/*
Peter Zijlstra
maintainerRe: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 14 lines · Peter Zijlstra
Quoted 5 lines · Paul Mackerras
Quoted 3 lines · Peter Zijlstra
Ingo just pointed out that there is nothing there to close the race with attaching a counter.
Quoted 7 lines
Quoted 1 line
That is, you could end up attaching your counter to the wrong task.
Good point. Doing the unclone in
find_get_contextwould be a start, but the locking could get tricky (in fact we don't have any way to remove an inherited counter from a context, so we only have to worry about counters being attached). I'll work out a solution after I have digested your recent batch of patches.
Quoted 4 lines
I'm currently staring at something like the below, trying to find races etc.. ;-)
attach vs destroy vs flip
Hmm, it appears we only unclone ctx on remove, not attach.. how odd..
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
In reply to Peter ZijlstraQuoted 2 lines · Peter Zijlstra
I'm currently staring at something like the below, trying to find races etc.. ;-)
[snip]
Review
Quoted 4 lines
(quoted diff) +2
-0next_ctx = next->perf_counter_ctxp; if (next_ctx && context_equiv(ctx, next_ctx)) { + ctx->task = NULL; + next_ctx->task = NULL;
Trouble is, ctx->task == NULL is used as an indication that this is a per-cpu context in various places.
Also, in find_get_context, we have a lifetime problem because *ctx could get swapped and then freed underneath us immediately after we read task->perf_counter_ctxp. So we need a lock in the task_struct that stops sched_out from swapping the context underneath us. That led me to the patch below, which I'm about to test... :) It does the unclone in find_get_context; we don't actually need it on remove because we have no way to remove an inherited counter from a task without the task exiting.
Paul.
include/linux/init_task.h +2 -0
@@ -110,6 +110,8 @@ extern struct cred init_cred;#ifdef CONFIG_PERF_COUNTERS # define INIT_PERF_COUNTERS (tsk) \ + .perf_counter_ctx_lock = \ + __SPIN_LOCK_UNLOCKED(tsk.perf_counter_ctx_lock), \ .perf_counter_mutex = \ __MUTEX_INITIALIZER(tsk.perf_counter_mutex), \ .perf_counter_list = LIST_HEAD_INIT(tsk.perf_counter_list),
include/linux/perf_counter.h +2 -2
@@ -531,8 +531,8 @@ struct perf_counter_context {* been cloned (inherited) from a common ancestor. */ struct perf_counter_context *parent_ctx; -u32 parent_gen; -u32 generation; +u64 parent_gen; +u64 generation; };/**
include/linux/sched.h +1 -0
@@ -1389,6 +1389,7 @@ struct task_struct {#endif #ifdef CONFIG_PERF_COUNTERS struct perf_counter_context *perf_counter_ctxp; +spinlock_t perf_counter_ctx_lock;struct mutex perf_counter_mutex;struct list_head perf_counter_list;#endif
kernel/perf_counter.c +67 -33
216 lines not on this page · raw message
Re: [PATCH 2/2] perf_counter: optimize context switch between identical inherited contexts
Quoted 4 lines · Paul Mackerras
Quoted 2 lines · Peter Zijlstra
I'm currently staring at something like the below, trying to find races etc.. ;-)
Quoted 8 lines
[snip]
kernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task,
Quoted 4 lines
kernel/perf_counter.c +2
-0next_ctx = next->perf_counter_ctxp;if (next_ctx &&context_equiv (ctx, next_ctx)) { + ctx->task =NULL ; + next_ctx->task =NULL ;Trouble is,
ctx->task== NULL is used as an indication that this is a per-cpu context in various places.
Yeah, already realized that, its enough to simply put them to the new task, before flipping the context pointers.
Quoted 8 lines
Also, in
find_get_context, we have a lifetime problem because *ctx could get swapped and then freed underneath us immediately after we readtask->perf_counter_ctxp. So we need a lock in thetask_structthat stopssched_outfrom swapping the context underneath us. That led me to the patch below, which I'm about to test... :) It does the unclone infind_get_context; we don't actually need it on remove because we have no way to remove an inherited counter from a task without the task exiting.
Right, I was trying to solve the lifetime issues with RCU and refcount tricks and the races with ordering, instead of adding another lock.
Quoted 22 lines
kernel/perf_counter.c +16
-0@@ -932,12 +927,32 @@ void perf_counter_task_sched_out(struct task_struct *task, regs =task_pt_regs (task);perf_swcounter_event (PERF_COUNT_CONTEXT_SWITCHES ,1 ,1 , regs,0 ); +/* +* Note: task->perf_counter_ctxp and next->perf_counter_ctxp +* can't change underneath us here if we see them non-NULL, +* because this is the only place where we change which +* context a task points to, and the scheduler will ensure +* that this code isn't being called for either of these tasks +* on any other cpu at the same time. +*/ next_ctx = next->perf_counter_ctxp;if (next_ctx &&context_equiv (ctx, next_ctx)) { +/* +* Lock the contexts of both the old and new tasks so that we +* don't change things underneath find_get_context etc. +* We don't need to be careful about the order in which we +* lock the tasks because no other cpu could be trying to lock +* both of these tasks -- this is the only place where we lock +* two tasks. +*/
kernel/perf_counter.c void perf_counter_task_sched_out(struct task_struct *task, · line 957
Quoted 10 lines
kernel/perf_counter.c +4
-0+spin_lock (&task->perf_counter_ctx_lock); +spin_lock (&next->perf_counter_ctx_lock); task->perf_counter_ctxp = next_ctx; next->perf_counter_ctxp = ctx; ctx->task = next; next_ctx->task = task; +spin_unlock (&next->perf_counter_ctx_lock); +spin_unlock (&task->perf_counter_ctx_lock);return ; }
FWIW that nested lock will make lockdep complain -- it can't deadlock since we're under rq->lock and the tasks can't be stolen from the rq in that case. So you can silence lockdep by using spin_lock_nested_lock()
kernel/perf_counter.c static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu) · line 1088
Quoted 8 lines
kernel/perf_counter.c +1
-0@@ -1067,6 +1082,7 @@ static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)__perf_counter_sched_in (ctx, cpuctx, cpu); } +// XXX doesn't do inherited counters too? int perf_counter_task_enable (void ) {struct perf_counter *counter;
Good point,.. perf_counter_for_each_child(counter, perf_counter_disable) should fix that I think.
kernel/perf_counter.c static int perf_release(struct inode *inode, struct file *file) · line 1395
Quoted 9 lines
kernel/perf_counter.c +1
-1@@ -1360,7 +1389,7 @@ static int perf_release(struct inode *inode, struct file *file)put_task_struct (counter->owner);free_counter (counter); -put_context (ctx); +put_context (ctx);// XXX use after free? return 0 ; }
don't htink so, but will have a look.
kernel/perf_counter.c static struct perf_counter_context *find_get_context(pid_t pid, int cpu) · line 1295
Quoted 6 lines · Paul Mackerras
kernel/perf_counter.c +6
-0+/* +* Make sure other cpus see correct values for *ctx +* once task->perf_counter_ctxp is visible to them. +*/ +smp_wmb (); + tctx =cmpxchg (&task->perf_counter_ctxp,NULL , ctx);
Documentation/memory-barriers.txt:
Quoted 6 lines
Any atomic operation that modifies some state in memory and returns information about the state (old or new) implies an SMP-conditional general memory barrier (
smp_mb()) on each side of the actual operation (with the exception of explicit lock operations, described later). These include:cmpxchg();
Quoted 9 lines · Peter Zijlstra
kernel/perf_counter.c static struct perf_counter_context *find_get_context(pid_t pid, int cpu) · line 1295
Quoted 6 lines · Paul Mackerras
kernel/perf_counter.c +6
-0+/* +* Make sure other cpus see correct values for *ctx +* once task->perf_counter_ctxp is visible to them. +*/ +smp_wmb (); + tctx =cmpxchg (&task->perf_counter_ctxp,NULL , ctx);
Quoted 8 lines
Documentation/memory-barriers.txt:Quoted 6 lines
Any atomic operation that modifies some state in memory and returns information about the state (old or new) implies an SMP-conditional general memory barrier (
smp_mb()) on each side of the actual operation (with the exception of explicit lock operations, described later). These include:cmpxchg();
OK, fair enough, I'll take it out.
Paul.
[tip:perfcounters/core] perf_counter: Dynamically allocate tasks' perf_counter_context struct
In reply to Paul MackerrasCommit-ID: a63eaf34ae60bdb067a354cc8def2e8f4a01f5f4 Gitweb: http://git.kernel.org/tip/a63eaf34ae60bdb067a354cc8def2e8f4a01f5f4 Author: Paul Mackerras <[email protected]> AuthorDate: Fri, 22 May 2009 14:17:31 +1000 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 22 May 2009 12:18:19 +0200
perf_counter: Dynamically allocate tasks' perf_counter_context struct
This replaces the struct perf_counter_context in the task_struct with a pointer to a dynamically allocated perf_counter_context struct. The main reason for doing is this is to allow us to transfer a perf_counter_context from one task to another when we do lazy PMU switching in a later patch.
This has a few side-benefits: the task_struct becomes a little smaller, we save some memory because only tasks that have perf_counters attached get a perf_counter_context allocated for them, and we can remove the inclusion of <linux/perf_counter.h> in sched.h, meaning that we don't end up recompiling nearly everything whenever perf_counter.h changes.
The perf_counter_context structures are reference-counted and freed when the last reference is dropped. A context can have references from its task and the counters on its task. Counters can outlive the task so it is possible that a context will be freed well after its task has exited.
Contexts are allocated on fork if the parent had a context, or otherwise the first time that a per-task counter is created on a task. In the latter case, we set the context pointer in the task struct locklessly using an atomic compare-and-exchange operation in case we raced with some other task in creating a context for the subject task.
This also removes the task pointer from the perf_counter struct. The task pointer was not used anywhere and would make it harder to move a context from one task to another. Anything that needed to know which task a counter was attached to was already using counter->ctx->task.
The __perf_counter_init_context function moves up in perf_counter.c so that it can be called from find_get_context, and now initializes the refcount, but is otherwise unchanged.
We were potentially calling list_del_counter twice: once from __perf_counter_exit_task when the task exits and once from __perf_counter_remove_from_context when the counter's fd gets closed. This adds a check in list_del_counter so it doesn't do anything if the counter has already been removed from the lists.
Since perf_counter_task_sched_in doesn't do anything if the task doesn't have a context, and leaves cpuctx->task_ctx = NULL, this adds code to __perf_install_in_context to set cpuctx->task_ctx if necessary, i.e. in the case where the current task adds the first counter to itself and thus creates a context for itself.
This also adds similar code to __perf_counter_enable to handle a similar situation which can arise when the counters have been disabled using prctl; that also leaves cpuctx->task_ctx = NULL.
[ Impact: refactor counter context management to prepare for new feature ]
LKML-Reference: <[email protected]>
--- arch/x86/kernel/apic/apic.c | 1 + include/linux/init_task.h | 13 --- include/linux/perf_counter.h | 4 +- include/linux/sched.h | 6 +- kernel/exit.c | 3 +- kernel/fork.c | 1 + kernel/perf_counter.c | 218 ++++++++++++++++++++++++++---------------- 7 files changed, 145 insertions(+), 101 deletions(-)
arch/x86/kernel/apic/apic.c +1 -0
@@ -14,6 +14,7 @@* Mikael Pettersson : PM converted to driver model. */ +#include <linux/perf_counter.h> #include <linux/kernel_stat.h> #include <linux/mc146818rtc.h> #include <linux/acpi_pmtmr.h>
include/linux/init_task.h +0 -13
@@ -108,18 +108,6 @@ extern struct group_info init_groups;extern struct cred init_cred; -#ifdef CONFIG_PERF_COUNTERS -# define INIT_PERF_COUNTERS (tsk) \ - .perf_counter_ctx.counter_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.counter_list), \ - .perf_counter_ctx.event_list = \ - LIST_HEAD_INIT(tsk.perf_counter_ctx.event_list), \ - .perf_counter_ctx.lock = \ - __SPIN_LOCK_UNLOCKED(tsk.perf_counter_ctx.lock), -#else -# define INIT_PERF_COUNTERS (tsk) -#endif -/* * INIT_TASK is used to set up the first task table, touch at * your own risk!. Base=0, limit=0x1fffff (=2MB)
@@ -183,7 +171,6 @@ extern struct cred init_cred; }, \ .dirties = INIT_PROP_LOCAL_SINGLE(dirties), \INIT_IDS \ -INIT_PERF_COUNTERS (tsk) \ INIT_TRACE_IRQFLAGS \INIT_LOCKDEP \INIT_FTRACE_GRAPH \
include/linux/perf_counter.h +1 -3
@@ -449,7 +449,6 @@ struct perf_counter {struct hw_perf_counter hw;struct perf_counter_context *ctx; -struct task_struct *task;struct file *filp;struct perf_counter *parent;
@@ -498,7 +497,6 @@ struct perf_counter {* Used as a container for task counters and CPU counters as well: */ struct perf_counter_context { -#ifdef CONFIG_PERF_COUNTERS /* * Protect the states of the counters in the list, * nr_active, and the list:
@@ -516,6 +514,7 @@ struct perf_counter_context {int nr_counters;int nr_active;int is_active; +atomic_t refcount;struct task_struct *task;/*
@@ -523,7 +522,6 @@ struct perf_counter_context {*/ u64 time;u64 timestamp; -#endif };/**
include/linux/sched.h +4 -2
@@ -71,7 +71,6 @@ struct sched_param {#include <linux/path.h> #include <linux/compiler.h> #include <linux/completion.h> -#include <linux/perf_counter.h> #include <linux/pid.h> #include <linux/percpu.h> #include <linux/topology.h>
@@ -99,6 +98,7 @@ struct robust_list_head;struct bio ;struct bts_tracer ;struct fs_struct ; +struct perf_counter_context ;/* * List of flags we want to share for kernel threads,
@@ -1387,7 +1387,9 @@ struct task_struct {struct list_head pi_state_list;struct futex_pi_state *pi_state_cache;#endif -struct perf_counter_context perf_counter_ctx; +#ifdef CONFIG_PERF_COUNTERS +struct perf_counter_context *perf_counter_ctxp; +#endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy;short il_next;
kernel/exit.c +2 -1
@@ -48,6 +48,7 @@#include <linux/tracehook.h> #include <linux/fs_struct.h> #include <linux/init_task.h> +#include <linux/perf_counter.h> #include <trace/sched.h> #include <asm/uaccess.h>
@@ -159,7 +160,7 @@ static void delayed_put_task_struct(struct rcu_head *rhp) struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);#ifdef CONFIG_PERF_COUNTERS -WARN_ON_ONCE (!list_empty (&tsk->perf_counter_ctx.counter_list)); +WARN_ON_ONCE (tsk->perf_counter_ctxp);#endif trace_sched_process_free (tsk);put_task_struct (tsk);
kernel/fork.c +1 -0
@@ -63,6 +63,7 @@#include <linux/fs_struct.h> #include <trace/sched.h> #include <linux/magic.h> +#include <linux/perf_counter.h> #include <asm/pgtable.h> #include <asm/pgalloc.h>
kernel/perf_counter.c +136 -82
440 lines not on this page · raw message