perf_events: add cgroup support (v6)
Subsystems (2)
PATCH 4/5 perf_events: add cgroup support (v6)
This kernel patch adds the ability to filter monitoring based on container groups (cgroups). This is for use in per-cpu mode only.
The cgroup to monitor is passed as a file descriptor in the pid argument to the syscall. The file descriptor must be opened to the cgroup name in the cgroup filesystem. For instance, if the cgroup name is foo and cgroupfs is mounted in /cgroup, then the file descriptor is opened to /cgroup/foo. Cgroup mode is activated by passing PERF_FLAG_PID_CGROUP in the flags argument to the syscall.
For instance to measure in cgroup foo on CPU1 assuming cgroupfs is mounted under /cgroup:
struct perf_event_attr attr;
int cgroup_fd, fd;
cgroup_fd = open("/cgroup/foo", O_RDONLY);
fd = perf_event_open(&attr, cgroup_fd, 1, -1, PERF_FLAG_PID_CGROUP); close(cgroup_fd);
---
include/linux/cgroup.h +1 -0
@@ -626,6 +626,7 @@ bool css_is_ancestor(struct cgroup_subsys_state *cg,/* Get id and depth of css */ unsigned short css_id (struct cgroup_subsys_state *css);unsigned short css_depth (struct cgroup_subsys_state *css); +struct cgroup_subsys_state *cgroup_css_from_dir (struct file *f,int id);#else /* !CONFIG_CGROUPS */
include/linux/cgroup_subsys.h +4 -0
@@ -65,4 +65,8 @@ SUBSYS(net_cls) SUBSYS(blkio)#endif +#ifdef CONFIG_PERF_CGROUPS +SUBSYS(perf) +#endif +/* */
include/linux/perf_event.h +26 -3
@@ -454,6 +454,7 @@ enum perf_callchain_context {#define PERF_FLAG_FD_NO_GROUP (1U << 0)#define PERF_FLAG_FD_OUTPUT (1U << 1) +#define PERF_FLAG_PID_CGROUP (1U << 2)/* pid=cgroup id, per-cpu mode only */ #ifdef __KERNEL__/*
@@ -461,6 +462,7 @@ enum perf_callchain_context {*/ #ifdef CONFIG_PERF_EVENTS +# include <linux/cgroup.h> # include <asm/perf_event.h> # include <asm/local64.h> #endif
@@ -702,6 +704,22 @@ struct swevent_hlist {#define PERF_ATTACH_GROUP 0x02#define PERF_ATTACH_TASK 0x04 +#ifdef CONFIG_PERF_CGROUPS +/* +* perf_cgroup_info keeps track of time_enabled for a cgroup. +* This is a per-cpu dynamically allocated data structure. +*/ +struct perf_cgroup_info { +u64 time; +u64 timestamp; +}; + +struct perf_cgroup { +struct cgroup_subsys_state css; +struct perf_cgroup_info *info;/* timing info, one per cpu */ +}; +#endif +/** * struct perf_event - performance event kernel representation: */
@@ -815,6 +833,10 @@ struct perf_event {struct event_filter *filter;#endif +#ifdef CONFIG_PERF_CGROUPS +struct perf_cgroup *cgrp;/* cgroup event is attach to */ +#endif +#endif /* CONFIG_PERF_EVENTS */ };
@@ -869,6 +891,7 @@ struct perf_event_context {u64 generation;int pin_count;struct rcu_head rcu_head; +int nr_cgroups;/* cgroup events present */ };/*
@@ -1022,11 +1045,11 @@ have_event:__perf_sw_event (event_id, nr, nmi, regs, addr); } -extern atomic_t perf_task_events; +extern atomic_t perf_sched_events;static inline void perf_event_task_sched_in (struct task_struct *task) { -COND_STMT (&perf_task_events,__perf_event_task_sched_in (task)); +COND_STMT (&perf_sched_events,__perf_event_task_sched_in (task)); } static inline
@@ -1034,7 +1057,7 @@ void perf_event_task_sched_out(struct task_struct *task, struct task_struct *nex {perf_sw_event (PERF_COUNT_SW_CONTEXT_SWITCHES ,1 ,1 ,NULL ,0 ); -COND_STMT (&perf_task_events,__perf_event_task_sched_out (task, next)); +COND_STMT (&perf_sched_events,__perf_event_task_sched_out (task, next)); }extern void perf_event_mmap (struct vm_area_struct *vma);
init/Kconfig +11 -0
@@ -1066,6 +1066,17 @@ config PERF_COUNTERS Say N if unsure. +config PERF_CGROUPS + bool "Enable per-cpu per-container group (cgroup) monitoring" + default y if CGROUPS + depends on PERF_EVENTS + help + This option extends the per-cpu mode to restrict monitoring to + threads which belong to the cgroup specificied and run on the + designated cpu. + + Say N if unsure. + config DEBUG_PERF_USE_VMALLOC default n bool "Debug: use vmalloc to back perf mmap() buffers"
kernel/cgroup.c +23 -0
@@ -4790,6 +4790,29 @@ css_get_next(struct cgroup_subsys *ss, int id,return ret; } +/* +* get corresponding css from file open on cgroupfs directory +*/ +struct cgroup_subsys_state *cgroup_css_from_dir (struct file *f,int id) +{ +struct cgroup *cgrp; +struct inode *inode; +struct cgroup_subsys_state *css; + + inode = f->f_dentry->d_inode; +/* check in cgroup filesystem dir */ +if (inode->i_op != &cgroup_dir_inode_operations) +return ERR_PTR (-EBADF ); + +if (id <0 || id >=CGROUP_SUBSYS_COUNT ) +return ERR_PTR (-EINVAL ); + +/* get cgroup */ + cgrp =__d_cgrp (f->f_dentry); + css = cgrp->subsys[id]; +return css ? css :ERR_PTR (-ENOENT ); +} +#ifdef CONFIG_CGROUP_DEBUG static struct cgroup_subsys_state *debug_create (struct cgroup_subsys *ss,struct cgroup *cont)
kernel/perf_event.c +543 -41
900 lines not on this page · raw message
Peter Zijlstra
maintainerQuoted 12 lines · Stephane Eranian
init/Kconfig +4
-0@@ -1066,6 +1066,17 @@ config PERF_COUNTERS Say N if unsure. +config PERF_CGROUPS + bool "Enable per-cpu per-container group (cgroup) monitoring" + default y if CGROUPS + depends on PERF_EVENTS
init/Kconfig config PERF_COUNTERS · line 1082
Quoted 10 lines
init/Kconfig +7
-0+ help + This option extends the per-cpu mode to restrict monitoring to + threads which belong to the cgroup specificied and run on the + designated cpu. + + Say N if unsure. + config DEBUG_PERF_USE_VMALLOC default n bool "Debug: use vmalloc to back perf mmap() buffers"
Usually the cgroup things live in: menuconfig CGROUPS.
Quoted 20 lines
kernel/cgroup.c +12
-0@@ -4790,6 +4790,29 @@ css_get_next(struct cgroup_subsys *ss, int id,return ret; } +/* +* get corresponding css from file open on cgroupfs directory +*/ +struct cgroup_subsys_state *cgroup_css_from_dir (struct file *f,int id) +{ +struct cgroup *cgrp; +struct inode *inode; +struct cgroup_subsys_state *css; + + inode = f->f_dentry->d_inode; +/* check in cgroup filesystem dir */ +if (inode->i_op != &cgroup_dir_inode_operations)
kernel/cgroup.c css_get_next(struct cgroup_subsys *ss, int id, · line 4814
Quoted 10 lines
kernel/cgroup.c +10
-0+return ERR_PTR (-EBADF ); + +if (id <0 || id >=CGROUP_SUBSYS_COUNT ) +return ERR_PTR (-EINVAL ); + +/* get cgroup */ + cgrp =__d_cgrp (f->f_dentry); + css = cgrp->subsys[id]; +return css ? css :ERR_PTR (-ENOENT ); +}
Since this paradigm was already in use it surprises me you have to add this function.. ?
Quoted 15 lines
kernel/cgroup.c +0
-0#ifdef CONFIG_CGROUP_DEBUG static struct cgroup_subsys_state *debug_create (struct cgroup_subsys *ss,struct cgroup *cont)kernel/perf_event.c +4
-0@@ -35,13 +35,22 @@#include <asm/irq_regs.h> +#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ + PERF_FLAG_FD_OUTPUT |\ + PERF_FLAG_PID_CGROUP) +
kernel/perf_event.c line 50
Quoted 10 lines
kernel/perf_event.c +3
-1enum event_type_t {EVENT_FLEXIBLE =0x1 ,EVENT_PINNED =0x2 ,EVENT_ALL =EVENT_FLEXIBLE |EVENT_PINNED , }; -atomic_t perf_task_events__read_mostly ; +/* perf_sched_events : >0 events exist +* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu +*/
That wants to be:
/*
* text goes here
*/
kernel/perf_event.c line 52
Quoted 2 lines
kernel/perf_event.c +2
-0+atomic_t perf_sched_events__read_mostly ; +static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
If its per-cpu, do we really need atomic_t ?
Quoted 18 lines
kernel/perf_event.c +4
-1static atomic_t nr_mmap_events__read_mostly ;static atomic_t nr_comm_events__read_mostly ;static atomic_t nr_task_events__read_mostly ;@@ -72,7 +81,10 @@ static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, enum event_type_t event_type);static void cpu_ctx_sched_in (struct perf_cpu_context *cpuctx, -enum event_type_t event_type); +enum event_type_t event_type, +struct task_struct *task,int css_sw); + +static u64 perf_event_time (struct perf_event *event);void __weak perf_event_print_debug (void ) { }@@ -86,6 +98,329 @@ static inline u64 perf_clock(void)return local_clock (); }
kernel/perf_event.c static inline u64 perf_clock(void) · line 109
Quoted 10 lines
kernel/perf_event.c +9
-0+#ifdef CONFIG_PERF_CGROUPS +static inline struct perf_cgroup * +perf_cgroup_from_task (struct task_struct *task) +{ +if (!task) +return NULL ; +return container_of (task_subsys_state (task, perf_subsys_id), + struct perf_cgroup, css); +}
Wouldn't it be nicer if the caller ensured to not call it for !task?
Quoted 14 lines
kernel/perf_event.c +14
-0+static struct perf_cgroup *perf_get_cgroup (int fd) +{ +struct cgroup_subsys_state *css; +struct file *file; +int fput_needed; + + file =fget_light (fd, &fput_needed); +if (!file) +return ERR_PTR (-EBADF ); + + css =cgroup_css_from_dir (file, perf_subsys_id); +if (!IS_ERR (css)) +css_get (css); +
kernel/perf_event.c static inline u64 perf_clock(void) · line 143
Quoted 10 lines
kernel/perf_event.c +10
-0+fput_light (file, fput_needed); + + return container_of(css, struct perf_cgroup, css); +} + +static inline void perf_put_cgroup (struct perf_event *event) +{ +if (event->cgrp) +css_put (&event->cgrp->css); +}
Bit asymmetric, you get a perf_cgroup, but you put a perf_event.
Quoted 6 lines
kernel/perf_event.c +6
-0+static inline void __update_css_time (struct perf_cgroup *cgrp) +{ +struct perf_cgroup_info *t; +u64 now; +int cpu =smp_processor_id (); +
kernel/perf_event.c static inline u64 perf_clock(void) · line 176
Quoted 10 lines
kernel/perf_event.c +10
-0+if (!cgrp) +return ; + + now =perf_clock (); + + t =per_cpu_ptr (cgrp->info, cpu); + + t->time += now - t->timestamp; + t->timestamp = now; +}
Most callers seem to already check for !cgrp, make that all and avoid the second conditional?
Quoted 44 lines
kernel/perf_event.c +44
-0+/* +* called from perf_event_ask_sched_out() conditional to jump label +*/ +void +perf_cgroup_switch (struct task_struct *task,struct task_struct *next) +{ +struct perf_cgroup *cgrp_out =perf_cgroup_from_task (task); +struct perf_cgroup *cgrp_in =perf_cgroup_from_task (next); +struct perf_cpu_context *cpuctx; +struct pmu *pmu; +/* +* if task is DEAD, then css_out is irrelevant, it has +* been changed to init_css in cgroup_exit() from do_exit(). +* Furthermore, perf_cgroup_exit_task(), has scheduled out +* all css constrained events, only unconstrained events +* remain. Therefore we need to reschedule based on css_in. +*/ +if (task->state !=TASK_DEAD && cgrp_out == cgrp_in) +return ; + +rcu_read_lock (); + +list_for_each_entry_rcu (pmu, &pmus, entry) { + + cpuctx =get_cpu_ptr (pmu->pmu_cpu_context); + +perf_pmu_disable (cpuctx->ctx.pmu); + +/* +* perf_cgroup_events says at least one +* context on this CPU has cgroup events. +* +* ctx->nr_cgroups reports the number of cgroup +* events for a context. Given there can be multiple +* PMUs, there can be multiple contexts. +*/ +if (cpuctx->ctx.nr_cgroups >0 ) { +/* +* schedule out everything we have +* task == DEAD: only unconstrained events +* task != DEAD: css constrained + unconstrained events +* +* We kick out all events (even if unconstrained) +* to allow the constrained events to be scheduled
kernel/perf_event.c static inline u64 perf_clock(void) · line 278
Quoted 10 lines
kernel/perf_event.c +10
-0+* based on their position in the event list (fairness) +*/ +cpu_ctx_sched_out (cpuctx,EVENT_ALL ); +/* +* reschedule css_in constrained + unconstrained events +*/ +cpu_ctx_sched_in (cpuctx,EVENT_ALL , next,1 ); + } + +perf_pmu_enable (cpuctx->ctx.pmu);
Do you leak a preemption count here? No matching put_cpu_ptr().
Since we're in the middle of a context switch, preemption is already disabled and it might be best to use this_cpu_ptr() instead of get_cpu_ptr(). That avoids the preemption bits.
Quoted 38 lines
kernel/perf_event.c +38
-0+ } + +rcu_read_unlock (); +} + +static inline void +perf_cgroup_exit_task (struct task_struct *task) +{ +struct perf_cpu_context *cpuctx; +struct pmu *pmu; +unsigned long flags; + +local_irq_save (flags); + +rcu_read_lock (); + +list_for_each_entry_rcu (pmu, &pmus, entry) { + + cpuctx =get_cpu_ptr (pmu->pmu_cpu_context); + +perf_pmu_disable (cpuctx->ctx.pmu); + +if (cpuctx->ctx.nr_cgroups >0 ) { +/* +* task is going to be detached from css. +* We cannot keep a reference on the css +* as it may disappear before we get to +* perf_cgroup_switch(). Thus, we remove +* all css constrained events. +* +* We do this by scheduling out everything +* we have, and then only rescheduling only +* the unconstrained events. Those can keep +* on counting. +* +* We re-examine the situation in the final +* perf_cgroup_switch() call for this task +* once we know the next task.
kernel/perf_event.c static inline u64 perf_clock(void) · line 326
Quoted 10 lines
kernel/perf_event.c +10
-0+*/ +cpu_ctx_sched_out (cpuctx,EVENT_ALL ); +/* +* task = NULL causes perf_cgroup_match() +* to match only unconstrained events +*/ +cpu_ctx_sched_in (cpuctx,EVENT_ALL ,NULL ,1 ); + } + +perf_pmu_enable (cpuctx->ctx.pmu);
Another preemption leak?
Quoted 19 lines
kernel/perf_event.c +9
-0+ } +rcu_read_unlock (); + +local_irq_restore (flags); +}@@ -246,6 +581,10 @@ static void update_context_time(struct perf_event_context *ctx)static u64 perf_event_time (struct perf_event *event) {struct perf_event_context *ctx = event->ctx; + +if (is_cgroup_event (event)) +return perf_cgroup_event_css_time (event); +return ctx ? ctx->time :0 ; }@@ -261,8 +600,10 @@ static void update_event_times(struct perf_event *event) event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
kernel/perf_event.c static void update_event_times(struct perf_event *event) · line 608
Quoted 10 lines
kernel/perf_event.c +4
-2return ; -if (ctx->is_active) - run_end =perf_event_time (event); +if (is_cgroup_event (event)) + run_end =perf_cgroup_event_css_time (event); +else if (ctx->is_active) + run_end = ctx->time;else run_end = event->tstamp_stopped;
So I guess the difference is that we want perf_cgroup_event_css_time() even when !active?
I'm afraid all this time accounting stuff is going to make my head explode.. could a comment help?
Quoted 4 lines
kernel/perf_event.c +0
-0@@ -322,6 +663,17 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)list_add_tail (&event->group_entry, list); }
kernel/perf_event.c list_add_event(struct perf_event *event, struct perf_event_context *ctx) · line 675
Quoted 10 lines
kernel/perf_event.c +10
-0+if (is_cgroup_event (event)) { + ctx->nr_cgroups++; +/* +* one more event: +* - that has cgroup constraint on event->cpu +* - that may need work on context switch +*/ +atomic_inc (&per_cpu (perf_cgroup_events, event->cpu)); +jump_label_inc (&perf_sched_events); + }
Ah, I guess this is why you're still using atomics, since another cpu can install the counters on the target cpu,. ok I guess that makes sense.
Quoted 30 lines
kernel/perf_event.c +9
-2list_add_rcu (&event->event_entry, &ctx->event_list);if (!ctx->nr_events)perf_pmu_rotate_start (ctx->pmu);@@ -368,6 +720,12 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) event->attach_state &= ~PERF_ATTACH_CONTEXT ; +if (is_cgroup_event (event)) { + ctx->nr_cgroups--; +atomic_dec (&per_cpu (perf_cgroup_events, event->cpu)); +jump_label_dec (&perf_sched_events); + } + ctx->nr_events--;if (event->attr.inherit_stat) ctx->nr_stat--;@@ -431,9 +789,10 @@ static void perf_group_detach(struct perf_event *event) }static inline int -event_filter_match (struct perf_event *event) +event_filter_match (struct perf_event *event,struct task_struct *task) { -return event->cpu ==-1 || event->cpu ==smp_processor_id (); +return (event->cpu ==-1 || event->cpu ==smp_processor_id ()) + &&perf_cgroup_match (event, task); } static void
kernel/perf_event.c event_sched_out(struct perf_event *event, · line 816
Quoted 10 lines
kernel/perf_event.c +2
-2@@ -450,8 +809,8 @@ event_sched_out(struct perf_event *event,* via read() for time_enabled, time_running: */ if (event->state ==PERF_EVENT_STATE_INACTIVE - && !event_filter_match (event)) { - delta = ctx->time - event->tstamp_stopped; + && !event_filter_match (event, current)) { + delta = tstamp - event->tstamp_stopped; event->tstamp_running += delta; event->tstamp_stopped = tstamp; }
Is that s/ctx->time/tstamp/ a missing hunk from patch 3?
Quoted 32 lines
kernel/perf_event.c +20
-1@@ -609,6 +968,7 @@ static void __perf_event_disable(void *info)*/ if (event->state >=PERF_EVENT_STATE_INACTIVE ) {update_context_time (ctx); +update_css_time_from_event (event);update_group_times (event);if (event == event->group_leader)group_sched_out (event, cpuctx, ctx);@@ -696,7 +1056,35 @@ event_sched_in(struct perf_event *event, event->tstamp_running += tstamp - event->tstamp_stopped; - event->shadow_ctx_time = tstamp - ctx->timestamp; +/* +* use the correct time source for the time snapshot +* +* We could get by without this by leveraging the +* fact that to get to this function, the caller +* has most likely already called update_context_time() +* and update_css_time_xx() and thus both timestamp +* are identical (or very close). Given that tstamp is, +* already adjusted for cgroup, we could say that: +* tstamp - ctx->timestamp +* is equivalent to +* tstamp - cgrp->timestamp. +* +* Then, in perf_output_read(), the calculation would +* work with no changes because: +* - event is guaranteed scheduled in +* - no scheduled out in between +* - thus the timestamp would be the same +*
kernel/perf_event.c event_sched_in(struct perf_event *event, · line 1087
Quoted 10 lines
kernel/perf_event.c +10
-0+* But this is a bit hairy. +* +* So instead, we have an explicit cgroup call to remain +* within the time time source all along. We believe it +* is cleaner and simpler to understand. +*/ +if (is_cgroup_event (event)) +perf_cgroup_set_shadow_time (event, tstamp); +else + event->shadow_ctx_time = tstamp - ctx->timestamp;
How about we make perf_set_shadow_time() and hide all this in there?
Quoted 13 lines
kernel/perf_event.c +2
-0@@ -5289,6 +5719,7 @@ unlock:static struct perf_event *perf_event_alloc (struct perf_event_attr *attr,int cpu,struct task_struct *task, +int cgrp_fd,int flags,struct perf_event *group_leader,struct perf_event *parent_event,perf_overflow_handler_t overflow_handler)@@ -5302,6 +5733,14 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,if (!event)return ERR_PTR (-ENOMEM ); +if (flags &PERF_FLAG_PID_CGROUP ) {
kernel/perf_event.c perf_event_alloc(struct perf_event_attr *attr, int cpu, · line 5746
Quoted 10 lines
kernel/perf_event.c +7
-0+ err =perf_connect_cgroup (cgrp_fd, event, attr, group_leader); +if (err) { +kfree (event); +return ERR_PTR (err); + } + } +/* * Single events are their own group leaders, with an * empty sibling list:
Hrm,. that isn't particularly pretty,.. why do we have to do this in perf_event_alloc()? Can't we do this in the syscall after perf_event_alloc() returns?
Quoted 27 lines · Peter Zijlstra
Quoted 25 lines · Stephane Eranian
kernel/cgroup.c +17
-0@@ -4790,6 +4790,29 @@ css_get_next(struct cgroup_subsys *ss, int id, return ret; } +/* +* get corresponding css from file open on cgroupfs directory +*/ +struct cgroup_subsys_state *cgroup_css_from_dir (struct file *f,int id) +{ + struct cgroup *cgrp; + struct inode *inode; + struct cgroup_subsys_state *css; + + inode = f->f_dentry->d_inode; + /* check in cgroup filesystem dir */ + if (inode->i_op != &cgroup_dir_inode_operations) + return ERR_PTR(-EBADF); + + if (id < 0 || id >= CGROUP_SUBSYS_COUNT) + return ERR_PTR(-EINVAL); +
Quoted 9 lines
kernel/cgroup.c css_get_next(struct cgroup_subsys *ss, int id, · line 4814
Quoted 5 lines
kernel/cgroup.c +5
-0+ /* get cgroup */ + cgrp = __d_cgrp(f->f_dentry); + css = cgrp->subsys[id]; + return css ? css : ERR_PTR(-ENOENT); +}Since this paradigm was already in use it surprises me you have to add this function.. ?
Well, I could not find one. If anybody knows of one, I'll be able to check it out.
Quoted 31 lines
kernel/cgroup.c css_get_next(struct cgroup_subsys *ss, int id,
Quoted 9 lines
kernel/cgroup.c +9
-0+#ifdef CONFIG_PERF_CGROUPS +static inline struct perf_cgroup * +perf_cgroup_from_task (struct task_struct *task) +{ + if (!task) + return NULL; + return container_of(task_subsys_state(task, perf_subsys_id), + struct perf_cgroup, css); +}Wouldn't it be nicer if the caller ensured to not call it for !task?
Quoted 18 lines
kernel/cgroup.c +18
-0+static struct perf_cgroup *perf_get_cgroup (int fd) +{ + struct cgroup_subsys_state *css; + struct file *file; + int fput_needed; + + file = fget_light(fd, &fput_needed); + if (!file) + return ERR_PTR(-EBADF); + + css = cgroup_css_from_dir(file, perf_subsys_id); + if (!IS_ERR(css)) + css_get(css); + + fput_light(file, fput_needed); + + return container_of(css, struct perf_cgroup, css); +}
Quoted 9 lines
kernel/cgroup.c
Quoted 6 lines
kernel/cgroup.c +6
-0+ +static inline void perf_put_cgroup (struct perf_event *event) +{ + if (event->cgrp) + css_put(&event->cgrp->css); +}Bit asymmetric, you get a
perf_cgroup, but you put aperf_event.
Ok, I made this symmetrical now.
Quoted 12 lines
Quoted 12 lines
kernel/cgroup.c +11
-0+static inline void __update_css_time (struct perf_cgroup *cgrp) +{ + struct perf_cgroup_info *t; + u64 now; + int cpu = smp_processor_id(); + + if (!cgrp) + return; + + now = perf_clock(); +
Quoted 9 lines
kernel/cgroup.c
Quoted 5 lines
+ t =
per_cpu_ptr(cgrp->info, cpu);kernel/cgroup.c +4
-0+ + t->time += now - t->timestamp; + t->timestamp = now; +}Most callers seem to already check for !cgrp, make that all and avoid the second conditional?
Done.
Quoted 57 lines
Quoted 44 lines
kernel/cgroup.c +44
-0+/* +* called from perf_event_ask_sched_out() conditional to jump label +*/ +void +perf_cgroup_switch (struct task_struct *task,struct task_struct *next) +{ + struct perf_cgroup *cgrp_out = perf_cgroup_from_task(task); + struct perf_cgroup *cgrp_in = perf_cgroup_from_task(next); + struct perf_cpu_context *cpuctx; + struct pmu *pmu; + /* + * if task is DEAD, then css_out is irrelevant, it has + * been changed to init_css in cgroup_exit() from do_exit(). + * Furthermore, perf_cgroup_exit_task(), has scheduled out + * all css constrained events, only unconstrained events + * remain. Therefore we need to reschedule based on css_in. + */ + if (task->state != TASK_DEAD && cgrp_out == cgrp_in) + return; + + rcu_read_lock(); + + list_for_each_entry_rcu(pmu, &pmus, entry) { + + cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); + + perf_pmu_disable(cpuctx->ctx.pmu); + + /* + * perf_cgroup_events says at least one + * context on this CPU has cgroup events. + * + * ctx->nr_cgroups reports the number of cgroup + * events for a context. Given there can be multiple + * PMUs, there can be multiple contexts. + */ + if (cpuctx->ctx.nr_cgroups > 0) { + /* +* schedule out everything we have +* task == DEAD: only unconstrained events +* task != DEAD: css constrained + unconstrained events +* +* We kick out all events (even if unconstrained) +* to allow the constrained events to be scheduled kernel/cgroup.c
Quoted 10 lines
kernel/cgroup.c +10
-0+* based on their position in the event list (fairness) +*/ + cpu_ctx_sched_out(cpuctx, EVENT_ALL); + /* + * reschedule css_in constrained + unconstrained events + */ + cpu_ctx_sched_in(cpuctx, EVENT_ALL, next, 1); + } + + perf_pmu_enable(cpuctx->ctx.pmu);Do you leak a preemption count here? No matching
put_cpu_ptr().
Quoted 4 lines
Since we're in the middle of a context switch, preemption is already disabled and it might be best to use
this_cpu_ptr()instead ofget_cpu_ptr(). That avoids the preemption bits.
Done.
Quoted 37 lines
Quoted 37 lines
kernel/cgroup.c +37
-0+static inline void +perf_cgroup_exit_task (struct task_struct *task) +{ + struct perf_cpu_context *cpuctx; + struct pmu *pmu; + unsigned long flags; + + local_irq_save(flags); + + rcu_read_lock(); + + list_for_each_entry_rcu(pmu, &pmus, entry) { + + cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); + + perf_pmu_disable(cpuctx->ctx.pmu); + + if (cpuctx->ctx.nr_cgroups > 0) { + /* + * task is going to be detached from css. + * We cannot keep a reference on the css + * as it may disappear before we get to + * perf_cgroup_switch(). Thus, we remove + * all css constrained events. + * + * We do this by scheduling out everything + * we have, and then only rescheduling only + * the unconstrained events. Those can keep + * on counting. + * + * We re-examine the situation in the final + * perf_cgroup_switch() call for this task + * once we know the next task. + */ + cpu_ctx_sched_out(cpuctx, EVENT_ALL); + /* +* task = NULL causes perf_cgroup_match()
Quoted 9 lines
Quoted 6 lines
+ * to match only unconstrained events
+ */
+cpu_ctx_sched_in(cpuctx, EVENT_ALL, NULL, 1);
+ }
+
+perf_pmu_enable(cpuctx->ctx.pmu);Another preemption leak?
Done.
Quoted 16 lines
Quoted 16 lines
kernel/cgroup.c +4
-0@@ -246,6 +581,10 @@ static void update_context_time(struct perf_event_context *ctx) static u64 perf_event_time(struct perf_event *event) { struct perf_event_context *ctx = event->ctx; + + if (is_cgroup_event(event)) + return perf_cgroup_event_css_time(event); + return ctx ? ctx->time : 0; }@@ -261,8 +600,10 @@ static void update_event_times(struct perf_event *event) event->group_leader->state < PERF_EVENT_STATE_INACTIVE) return;
Quoted 12 lines
Quoted 8 lines
kernel/cgroup.c +4
-2- if (ctx->is_active) - run_end = perf_event_time(event); + if (is_cgroup_event(event)) + run_end = perf_cgroup_event_css_time(event); + else if (ctx->is_active) + run_end = ctx->time;else
run_end=event->tstamp_stopped;So I guess the difference is that we want
perf_cgroup_event_css_time()even when !active?
The difference is in the way time_enabled in accounted for in cgroup mode. time_enabled represent the time the event is enabled AND the monitored threads were active of the monitored CPU. Thus it is independent of the state of the context. A context may have cgroup and non-cgroup events attached to it. I have added a comment to explain that.
Quoted 4 lines
Quoted 4 lines
kernel/cgroup.c +0
-0@@ -322,6 +663,17 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) list_add_tail(&event->group_entry, list); }
Quoted 15 lines
kernel/cgroup.c list_add_event(struct perf_event *event, struct perf_event_context *ctx) · line 675
Quoted 10 lines
kernel/cgroup.c +10
-0+ if (is_cgroup_event(event)) { + ctx->nr_cgroups++; + /* + * one more event: + * - that has cgroup constraint on event->cpu + * - that may need work on context switch + */ + atomic_inc(&per_cpu(perf_cgroup_events, event->cpu)); + jump_label_inc(&perf_sched_events); + }Ah, I guess this is why you're still using atomics, since another cpu can install the counters on the target cpu,. ok I guess that makes sense.
YES!
Quoted 31 lines
Quoted 20 lines
kernel/cgroup.c +19
-1- event->shadow_ctx_time = tstamp - ctx->timestamp; + /* +* use the correct time source for the time snapshot +* +* We could get by without this by leveraging the +* fact that to get to this function, the caller +* has most likely already called update_context_time() +* and update_css_time_xx() and thus both timestamp +* are identical (or very close). Given that tstamp is, +* already adjusted for cgroup, we could say that: +* tstamp - ctx->timestamp +* is equivalent to +* tstamp - cgrp->timestamp. +* +* Then, in perf_output_read(), the calculation would +* work with no changes because: +* - event is guaranteed scheduled in +* - no scheduled out in between +* - thus the timestamp would be the same +* kernel/cgroup.c list_add_event(struct perf_event *event, struct perf_event_context *ctx)
Quoted 10 lines
kernel/cgroup.c +10
-0+* But this is a bit hairy. +* +* So instead, we have an explicit cgroup call to remain +* within the time time source all along. We believe it +* is cleaner and simpler to understand. +*/ + if (is_cgroup_event(event)) + perf_cgroup_set_shadow_time(event, tstamp); + else + event->shadow_ctx_time = tstamp - ctx->timestamp;
Quoted 3 lines
How about we make
perf_set_shadow_time()and hide all this in there?
Done.
Quoted 12 lines
Quoted 12 lines
kernel/cgroup.c +0
-0@@ -5289,6 +5719,7 @@ unlock:static struct
perf_event* perf_event_alloc(structperf_event_attr*attr, int cpu, structtask_struct*task,
+ intcgrp_fd, int flags,
structperf_event*group_leader, structperf_event*parent_event,perf_overflow_handler_toverflow_handler)kernel/cgroup.c +0
-0@@ -5302,6 +5733,14 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, if (!event) return ERR_PTR(-ENOMEM);
Quoted 17 lines
kernel/cgroup.c perf_event_alloc(struct perf_event_attr *attr, int cpu, · line 5746
Quoted 11 lines
kernel/cgroup.c +8
-0+ if (flags & PERF_FLAG_PID_CGROUP) { + err = perf_connect_cgroup(cgrp_fd, event, attr, group_leader); + if (err) { + kfree(event); + return ERR_PTR(err); + } + } + /** Single events are their own group leaders, with an * empty sibling list: Hrm,. that isn't particularly pretty,.. why do we have to do this in
perf_event_alloc()? Can't we do this in the syscall afterperf_event_alloc()returns?
Done.
Will be posting an updated version soon.
I also realized, I need to check how cgroup is handled for the SW events.