perf and jump_label bits
Subsystems (5)
cover perf and jump_label bits
Only compile tested, but should fix the pending hw_breakpoint issue and use jump_labels to optimize some of perf
RFC PATCH 3/7 perf, hw_breakpoint: Fix crash in hw_breakpoint creation
hw_breakpoint creation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.
With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.
This makes hw_breakpoints unhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in the event::hw bits for now...
--- include/linux/perf_event.h | 7 +++++++ kernel/hw_breakpoint.c | 8 ++++---- kernel/perf_event.c | 21 ++++++++++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-)
include/linux/perf_event.h +7 -0
@@ -536,6 +536,12 @@ struct hw_perf_event {struct {/* breakpoint */ struct arch_hw_breakpoint info;struct list_head bp_list; +/* +* Crufty hack to avoid the chicken and egg +* problem hw_breakpoint has with context +* creation and event initalization. +*/ +struct task_struct *bp_target; };#endif };
@@ -693,6 +699,7 @@ struct swevent_hlist {#define PERF_ATTACH_CONTEXT 0x01#define PERF_ATTACH_GROUP 0x02 +#define PERF_ATTACH_TASK 0x04/** * struct perf_event - performance event kernel representation:
kernel/hw_breakpoint.c +4 -4
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(i*/ static int task_bp_pinned (struct perf_event *bp,enum bp_type_idx type) { -struct perf_event_context *ctx = bp->ctx; +struct task_struct *tsk = bp->hw.bp_target;struct perf_event *iter;int count =0 ;list_for_each_entry (iter, &bp_task_head, hw.bp_list) { -if (iter->ctx == ctx &&find_slot_idx (iter) == type) +if (iter->hw.bp_target == tsk &&find_slot_idx (iter) == type) count +=hw_breakpoint_weight (iter); }
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots enum bp_type_idx type) {int cpu = bp->cpu; -struct task_struct *tsk = bp->ctx->task; +struct task_struct *tsk = bp->hw.bp_target;if (cpu >=0 ) { slots->pinned =per_cpu (nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bo int weight) {int cpu = bp->cpu; -struct task_struct *tsk = bp->ctx->task; +struct task_struct *tsk = bp->hw.bp_target;/* Pinned counter cpu profiling */ if (!tsk) {
kernel/perf_event.c +16 -5
@@ -5189,9 +5189,10 @@ struct pmu *perf_init_event(struct perf_*/ static struct perf_event *perf_event_alloc (struct perf_event_attr *attr,int cpu, -struct perf_event *group_leader, -struct perf_event *parent_event, -perf_overflow_handler_t overflow_handler) +struct task_struct *task, +struct perf_event *group_leader, +struct perf_event *parent_event, +perf_overflow_handler_t overflow_handler) {struct pmu *pmu;struct perf_event *event;
@@ -5233,6 +5234,15 @@ perf_event_alloc(struct perf_event_attr event->state =PERF_EVENT_STATE_INACTIVE ; +if (task) { + event->attach_state =PERF_ATTACH_TASK ; +/* +* hw_breakpoint is a bit difficult here.. +*/ +if (attr->type ==PERF_TYPE_BREAKPOINT ) + event->hw.bp_target = task; + } +if (!overflow_handler && parent_event) overflow_handler = parent_event->overflow_handler;
@@ -5493,7 +5503,7 @@ SYSCALL_DEFINE5(perf_event_open, } } - event =perf_event_alloc (&attr, cpu, group_leader,NULL ,NULL ); + event =perf_event_alloc (&attr, cpu, task, group_leader,NULL ,NULL );if (IS_ERR (event)) { err =PTR_ERR (event); goto err_task;
@@ -5662,7 +5672,7 @@ perf_event_create_kernel_counter(struct* Get the target context (task or percpu): */ - event =perf_event_alloc (attr, cpu,NULL ,NULL , overflow_handler); + event =perf_event_alloc (attr, cpu, task,NULL ,NULL , overflow_handler);if (IS_ERR (event)) { err =PTR_ERR (event); goto err;
@@ -5930,6 +5940,7 @@ inherit_event(struct perf_event *parent_ child_event =perf_event_alloc (&parent_event->attr, parent_event->cpu, + child, group_leader, parent_event,NULL );if (IS_ERR (child_event))
RFC PATCH 2/7 perf: Find task before event alloc
In reply to Peter ZijlstraSo that we can pass the task pointer to the event allocation, so that we can use task associated data during event initialization.
--- kernel/perf_event.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)
kernel/perf_event.c +12 -11
@@ -5485,10 +5485,18 @@ SYSCALL_DEFINE5(perf_event_open, group_leader =NULL ; } +if (pid !=-1 ) { + task =find_lively_task_by_vpid (pid); +if (IS_ERR (task)) { + err =PTR_ERR (task); + goto err_group_fd; + } + } + event =perf_event_alloc (&attr, cpu, group_leader,NULL ,NULL );if (IS_ERR (event)) { err =PTR_ERR (event); - goto err_fd; + goto err_task; }/*
@@ -5520,21 +5528,13 @@ SYSCALL_DEFINE5(perf_event_open, } } -if (pid !=-1 ) { - task =find_lively_task_by_vpid (pid); -if (IS_ERR (task)) { - err =PTR_ERR (task); - goto err_group_fd; - } - } -/* * Get the target context (task or percpu): */ ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_task; + goto err_alloc; }/*
@@ -5630,12 +5630,13 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_alloc: +free_event (event); err_task:if (task)put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed); -free_event (event); err_fd:put_unused_fd (event_fd);return err;
RFC PATCH 7/7 perf: Optimize sw events
In reply to Peter Zijlstra--- include/linux/perf_event.h | 16 +++++++++------- kernel/perf_event.c | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-)
include/linux/perf_event.h +9 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event); +return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }extern void perf_event_mmap (struct vm_area_struct *vma);
kernel/perf_event.c +2 -2
@@ -4603,7 +4603,7 @@ static void sw_perf_event_destroy(structWARN_ON (event->parent); -atomic_dec (&perf_swevent_enabled[event_id]); +jump_label_dec (&perf_swevent_enabled[event_id]);swevent_hlist_put (event); }
@@ -4633,7 +4633,7 @@ static int perf_swevent_init(struct perfif (err)return err; -atomic_inc (&perf_swevent_enabled[event_id]); +jump_label_inc (&perf_swevent_enabled[event_id]); event->destroy = sw_perf_event_destroy; }
RFC PATCH 6/7 perf: use jump_label to optimize the scheduler hooks
In reply to Peter Zijlstra--- include/linux/perf_event.h | 27 +++++++++++++++++++++++++-- kernel/perf_event.c | 24 +++++++++--------------- 2 files changed, 34 insertions(+), 17 deletions(-)
include/linux/perf_event.h +25 -2
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {#include <linux/ftrace.h> #include <linux/cpu.h> #include <linux/irq_work.h> +#include <linux/jump_label_ref.h> #include <asm/atomic.h> #include <asm/local.h>
@@ -893,8 +894,30 @@ struct perf_output_handle {extern int perf_pmu_register (struct pmu *pmu);extern void perf_pmu_unregister (struct pmu *pmu); -extern void perf_event_task_sched_in (struct task_struct *task); -extern void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next); +extern void __perf_event_task_sched_in (struct task_struct *task); +extern void __perf_event_task_sched_out (struct task_struct *task,struct task_struct *next); + +extern atomic_t perf_task_events; + +static inline void perf_event_task_sched_in (struct task_struct *task) +{ +JUMP_LABEL (&perf_task_events, have_events); +return ; + +have_events: +__perf_event_task_sched_in (task); +} + +static inline +void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next) +{ +JUMP_LABEL (&perf_task_events, have_events); +return ; + +have_events: +__perf_event_task_sched_out (task, next); +} +extern int perf_event_init_task (struct task_struct *child);extern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);
kernel/perf_event.c +9 -15
@@ -34,7 +34,7 @@#include <asm/irq_regs.h> -static atomic_t nr_events__read_mostly ; +atomic_t perf_task_events__read_mostly ;static atomic_t nr_mmap_events__read_mostly ;static atomic_t nr_comm_events__read_mostly ;static atomic_t nr_task_events__read_mostly ;
@@ -1251,8 +1251,8 @@ void perf_event_context_sched_out(struct* accessing the event control register. If a NMI hits, then it will * not restart the event. */ -void perf_event_task_sched_out (struct task_struct *task, -struct task_struct *next) +void __perf_event_task_sched_out (struct task_struct *task, +struct task_struct *next) {int ctxn;
@@ -1280,14 +1280,6 @@ static void task_ctx_sched_out(struct pe/* * Called with IRQs disabled */ -static void __perf_event_task_sched_out (struct perf_event_context *ctx) -{ -task_ctx_sched_out (ctx,EVENT_ALL ); -} - -/* -* Called with IRQs disabled -*/ static void cpu_ctx_sched_out (struct perf_cpu_context *cpuctx,enum event_type_t event_type) {
@@ -1434,7 +1426,7 @@ void perf_event_context_sched_in(struct* accessing the event control register. If a NMI hits, then it will * keep the event running. */ -void perf_event_task_sched_in (struct task_struct *task) +void __perf_event_task_sched_in (struct task_struct *task) {struct perf_event_context *ctx;int ctxn;
@@ -2150,7 +2142,8 @@ static void free_event(struct perf_eventirq_work_sync (&event->pending);if (!event->parent) { -atomic_dec (&nr_events); +if (event->attach_state &PERF_ATTACH_TASK ) +jump_label_dec (&perf_task_events);if (event->attr.mmap || event->attr.mmap_data)atomic_dec (&nr_mmap_events);if (event->attr.comm)
@@ -5286,7 +5279,8 @@ perf_event_alloc(struct perf_event_attr event->pmu = pmu;if (!event->parent) { -atomic_inc (&nr_events); +if (event->attach_state &PERF_ATTACH_TASK ) +jump_label_inc (&perf_task_events);if (event->attr.mmap || event->attr.mmap_data)atomic_inc (&nr_mmap_events);if (event->attr.comm)
@@ -5781,7 +5775,7 @@ static void perf_event_exit_task_context* our context. */ child_ctx = child->perf_event_ctxp[ctxn]; -__perf_event_task_sched_out (child_ctx); +task_ctx_sched_out (child_ctx,EVENT_ALL );/* * Take the context lock here so that if find_get_context is
RFC PATCH 1/7 perf: Fix task refcount issues
In reply to Peter ZijlstraCurrently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.
The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.
--- kernel/perf_event.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
kernel/perf_event.c +4 -3
@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct } } -put_task_struct (task);return ctx; errout: -put_task_struct (task);return ERR_PTR (err); }
@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open, ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_group_fd; + goto err_task; }/*
@@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_task: +if (task) +put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed);free_event (event);
RFC PATCH 4/7 jump_label: More consitent naming
In reply to Peter ZijlstraNow that there's only a few users around, rename things.
--- include/linux/jump_label.h | 8 ++++---- kernel/tracepoint.c | 6 +++--- lib/dynamic_debug.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/jump_label.h +4 -4
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned lextern void jump_label_apply_nops (struct module *mod);extern int jump_label_text_reserved (void *start,void *end); -#define enable_jump_label (key) \ +#define jump_label_enable (key) \ jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE); -#define disable_jump_label (key) \ +#define jump_label_disable (key) \ jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);#else
@@ -39,12 +39,12 @@ do { \ goto label; \ } while (0) -#define enable_jump_label (cond_var) \ +#define jump_label_enable (cond_var) \ do { \ *(cond_var) = 1; \ } while (0) -#define disable_jump_label (cond_var) \ +#define jump_label_disable (cond_var) \ do { \ *(cond_var) = 0; \ } while (0)
kernel/tracepoint.c +3 -3
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracep*/ rcu_assign_pointer (elem->funcs, (*entry)->funcs);if (!elem->state && active) { -enable_jump_label (&elem->state); +jump_label_enable (&elem->state); elem->state = active; }else if (elem->state && !active) { -disable_jump_label (&elem->state); +jump_label_disable (&elem->state); elem->state = active; } }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tr elem->unregfunc ();if (elem->state) { -disable_jump_label (&elem->state); +jump_label_disable (&elem->state); elem->state =0 ; }rcu_assign_pointer (elem->funcs,NULL );
lib/dynamic_debug.c +2 -2
@@ -142,9 +142,9 @@ static void ddebug_change(const struct d dt->num_enabled++; dp->flags = newflags;if (newflags) { -enable_jump_label (&dp->enabled); +jump_label_enable (&dp->enabled); }else { -disable_jump_label (&dp->enabled); +jump_label_disable (&dp->enabled); }if (verbose) printk(KERN_INFO
RFC PATCH 5/7 jump_label: atomic_t interface
In reply to Peter ZijlstraAdd an interface to allow usage of jump_labels with atomic counters
--- include/linux/jump_label_ref.h | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+)
include/linux/jump_label_ref.h new file +44 -0
@@ -0,0 +1,44 @@ +#ifndef _LINUX_JUMP_LABEL_REF_H +#define _LINUX_JUMP_LABEL_REF_H + +#include <linux/jump_label.h> +#include <asm/atomic.h> + +#ifdef HAVE_JUMP_LABEL + +static inline void jump_label_inc (atomic_t *key) +{ +if (atomic_add_return (1 , key) ==1 ) +jump_label_enable (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +if (atomic_dec_and_test (key)) +jump_label_disable (key); +} + +#else /* !HAVE_JUMP_LABEL */ + +static inline void jump_label_inc (atomic_t *key) +{ +atomic_inc (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +atomic_dec (key); +} + +#undef JUMP_LABEL +#define JUMP_LABEL (key, label) \ +do { \ + if (unlikely(__builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(key), atomic_t *), \ + atomic_read((atomic_t *)(key)), *(key)))) \ + goto label; \ +} while (0) + +#endif /* HAVE_JUMP_LABEL */ + +#endif /* _LINUX_JUMP_LABEL_REF_H */
Peter Zijlstra
maintainerRe: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 18 lines · Peter Zijlstra
Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +2
-7@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event);
include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028
Quoted 10 lines
include/linux/perf_event.h +7
-0+return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }
OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:
In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
The relevant snippet from the .i file reads:
static inline voidperf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) {
struct pt_regs hot_regs;
do { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table, \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : : have_event); } while (0); return;
have_event:
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
}
__perf_sw_event(event_id, nr, nmi, regs, addr);
}
Anybody got any clue as to why this goes splat?
I tried both:
gcc (GCC) 4.4.4 20100630 (Red Hat 4.4.4-10)
and
x86_64-linux-gcc (GCC) 4.5.1
Peter Zijlstra
maintainerRe: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation
In reply to Frederic WeisbeckerQuoted 7 lines · Frederic Weisbecker
Quoted 1 line
+#define
PERF_ATTACH_TASK0x04Thanks, the patch looks good.
I'm just not sure about the point of this flag...
Ah, see 6/7..
Peter Zijlstra
maintainerRe: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Frederic WeisbeckerQuoted 7 lines · Frederic Weisbecker
Quoted 2 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra <[email protected]>
Shall we get it compiling first? ;-)
Re: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 2 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra <[email protected]>
Re: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 57 lines · Peter Zijlstra
Quoted 18 lines
Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +2
-7@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event);include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028
Quoted 10 lines
include/linux/perf_event.h +7
-0+return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }OK, so it appears I only compile tested this bit without
jump_labelsupport, with that bit added back this horribly fails to compile like:In file included from
/usr/src/linux-2.6/arch/x86/mm/fault.c:13:/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints/usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’The relevant snippet from the .i file reads:
static inline void
perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr){
structpt_regshot_regs;do { asm goto("1:" ".byte
0xe9\n\t .long 0\n\t" ".pushsection__jump_table, \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : :have_event); } while (0); return;
have_event:
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs =&hot_regs;
}
__perf_sw_event(event_id, nr, nmi, regs, addr);
}
Quoted 2 lines
Anybody got any clue as to why this goes splat?
Couple issues here. The key value: "&perf_swevent_enabled[event_id]" is determined at run-time, so we can't associated the jump label with keys that are run-time dependent. The second thing is that there is array indexing, so essentially the jump label code would need to be smart enough to associated the correct index with each instance of 'perf_sw_event' - unfortunately its not that smart :(
So, two possible suggestions:
1)
I see that PERF_COUNT_SW_MAX is 9. So, we could explicitly split out the 9 cases into 9 separate inline functions where the keys would be:
&perf_swevent_enabled[PERF_COUNT_SW_CPU_CLOCK]
&perf_swevent_enabled[PERF_COUNT_SW_TASK_CLOCK]
this probably wouldn't look to ugly if there was a macro that took each software id, and spit out the appropriate inline. I see there aren't that many 'perf_sw_event()' calls either to update.
2)
If you could define a single global variable that was set if any of the indexes of the perf_swevent_enabled[] array were set. This potentially woulnd't be as efficient as #1 as it would create false positives.
thanks,
-Jason
Peter Zijlstra
maintainerRe: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra
Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +2
-7@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event);include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028
Quoted 10 lines
include/linux/perf_event.h +7
-0+return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }OK, so it appears I only compile tested this bit without
jump_labelsupport, with that bit added back this horribly fails to compile like:
Quoted 5 lines
In file included from
/usr/src/linux-2.6/arch/x86/mm/fault.c:13:/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints/usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.
I thought we had a __force_inline macro but I can't seem to find that.
Re: [RFC][PATCH 5/7] jump_label: atomic_t interface
In reply to Peter ZijlstraQuoted 21 lines · Peter Zijlstra
Add an interface to allow usage of
jump_labelswith atomic countersSigned-off-by Peter Zijlstra <[email protected]>--- include/linux/jump_label_ref.h | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+)Index:
linux-2.6/include/linux/jump_label_ref.h
===================================================================include/linux/jump_label_ref.h new file +9
-0@@ -0,0 +1,44 @@ +#ifndef _LINUX_JUMP_LABEL_REF_H +#define _LINUX_JUMP_LABEL_REF_H + +#include <linux/jump_label.h> +#include <asm/atomic.h> + +#ifdef HAVE_JUMP_LABEL + +static inline void jump_label_inc (atomic_t *key)
include/linux/jump_label_ref.h line 19
Quoted 10 lines
include/linux/jump_label_ref.h new file +10
-0+{ +if (atomic_add_return (1 , key) ==1 ) +jump_label_enable (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +if (atomic_dec_and_test (key)) +jump_label_disable (key); +}
Nice. I was trying to find a more self-explanatory name, like jump_label_get/put or jump_label_inc_enable/jump_label_dec_disable.
But in fact the current name looks good eventually.
Quoted 26 lines
include/linux/jump_label_ref.h new file +24
-0+#else /* !HAVE_JUMP_LABEL */ + +static inline void jump_label_inc (atomic_t *key) +{ +atomic_inc (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +atomic_dec (key); +} + +#undef JUMP_LABEL +#define JUMP_LABEL (key, label) \ +do { \ + if (unlikely(__builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(key), atomic_t *), \ + atomic_read((atomic_t *)(key)), *(key)))) \ + goto label; \ +} while (0) + +#endif /* HAVE_JUMP_LABEL */ + +#endif /* _LINUX_JUMP_LABEL_REF_H */
Re: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 35 lines · Peter Zijlstra
Quoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra
Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +2
-7@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event);include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028
Quoted 10 lines
include/linux/perf_event.h +7
-0+return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }OK, so it appears I only compile tested this bit without
jump_labelsupport, with that bit added back this horribly fails to compile like:
Quoted 9 lines
Quoted 5 lines
In file included from
/usr/src/linux-2.6/arch/x86/mm/fault.c:13:/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints/usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.
ah...cool! that should work. ignore my other suggestions...
thanks,
-Jason
Re: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 11 lines · Peter Zijlstra
Quoted 7 lines · Frederic Weisbecker
Quoted 2 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra <[email protected]>Shall we get it compiling first? ;-)
Why bother ;-)
Re: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation
In reply to Peter ZijlstraQuoted 39 lines · Peter Zijlstra
hw_breakpointcreation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.
This makes
hw_breakpointsunhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in theevent::hwbits for now...Signed-off-by Peter Zijlstra <[email protected]>Cc Frederic Weisbecker <[email protected]>--- include/linux/perf_event.h | 7 +++++++ kernel/hw_breakpoint.c | 8 ++++---- kernel/perf_event.c | 21 ++++++++++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-)Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +6
-0@@ -536,6 +536,12 @@ struct hw_perf_event {struct {/* breakpoint */ struct arch_hw_breakpoint info;struct list_head bp_list; +/* +* Crufty hack to avoid the chicken and egg +* problem hw_breakpoint has with context +* creation and event initalization. +*/ +struct task_struct *bp_target; };#endif };
include/linux/perf_event.h struct swevent_hlist { · line 702
Quoted 4 lines
include/linux/perf_event.h +1
-0@@ -693,6 +699,7 @@ struct swevent_hlist {#define PERF_ATTACH_CONTEXT 0x01#define PERF_ATTACH_GROUP 0x02 +#define PERF_ATTACH_TASK 0x04
Thanks, the patch looks good.
I'm just not sure about the point of this flag...
Re: [RFC][PATCH 1/7] perf: Fix task refcount issues
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Currently it looks like
find_lively_task_by_vpid()takes a task ref and relies onfind_get_context()to drop it.The problem is that
perf_event_create_kernel_counter()shouldn't be dropping task refs.Signed-off-by Peter Zijlstra <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Matt Helsley <[email protected]>
Looks good.
Quoted 41 lines
--- kernel/perf_event.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)Index:
linux-2.6/kernel/perf_event.c
===================================================================kernel/perf_event.c +4
-3@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct } } -put_task_struct (task);return ctx; errout: -put_task_struct (task);return ERR_PTR (err); }@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open, ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_group_fd; + goto err_task; }/* @@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_task: +if (task) +put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed);free_event (event);
Steven Rostedt
maintainerRe: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 4 lines · Peter Zijlstra
Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.
I thought we had a
__force_inlinemacro but I can't seem to find that.
Do you mean __always_inline?
See include/linux/compiler-gcc4.h
-- Steve
Peter Zijlstra
maintainerRe: [RFC][PATCH 7/7] perf: Optimize sw events
Quoted 10 lines · Steven Rostedt
Quoted 4 lines · Peter Zijlstra
Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.
I thought we had a
__force_inlinemacro but I can't seem to find that.Do you mean
__always_inline?See
include/linux/compiler-gcc4.h
Yeah, cyrillos already pointed that out..
Re: [RFC][PATCH 1/7] perf: Fix task refcount issues
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
Currently it looks like
find_lively_task_by_vpid()takes a task ref and relies onfind_get_context()to drop it.The problem is that
perf_event_create_kernel_counter()shouldn't be dropping task refs.
Doh.
Looks good.
Quoted 49 lines
Signed-off-by Peter Zijlstra <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Matt Helsley <[email protected]>--- kernel/perf_event.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)Index:
linux-2.6/kernel/perf_event.c
===================================================================kernel/perf_event.c +4
-4@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct } } -put_task_struct (task);return ctx; errout: -put_task_struct (task);return ERR_PTR (err); }@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open, ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_group_fd; + goto err_task; }/* @@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_task: +if (task) +put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed);free_event (event); --To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks
In reply to Peter ZijlstraQuoted 2 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra <[email protected]>
Cool!
Re: [RFC][PATCH 7/7] perf: Optimize sw events
In reply to Peter ZijlstraQuoted 41 lines · Peter Zijlstra
Quoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra
Index:
linux-2.6/include/linux/perf_event.h
===================================================================include/linux/perf_event.h +2
-7@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_regstatic inline void perf_sw_event (u32 event_id,u64 nr,int nmi,struct pt_regs *regs,u64 addr) { -if (atomic_read (&perf_swevent_enabled[event_id])) { -struct pt_regs hot_regs; +struct pt_regs hot_regs; -if (!regs) { -perf_fetch_caller_regs (&hot_regs); - regs = &hot_regs; - } -__perf_sw_event (event_id, nr, nmi, regs, addr); +JUMP_LABEL (&perf_swevent_enabled[event_id], have_event);include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028
Quoted 10 lines
include/linux/perf_event.h +7
-0+return ; + +have_event: +if (!regs) { +perf_fetch_caller_regs (&hot_regs); + regs = &hot_regs; } +__perf_sw_event (event_id, nr, nmi, regs, addr); }OK, so it appears I only compile tested this bit without
jump_labelsupport, with that bit added back this horribly fails to compile like:Quoted 5 lines
In file included from
/usr/src/linux-2.6/arch/x86/mm/fault.c:13:/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints/usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
Quoted 4 lines
Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.
I thought we had a
__force_inlinemacro but I can't seem to find that.
It's called __always_inline.
Thanks,
Ingo
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks
In reply to Peter Zijlstrainclude/linux/perf_event.h struct perf_output_handle { · line 909
Quoted 8 lines · Peter Zijlstra
include/linux/perf_event.h +8
-0+static inline void perf_event_task_sched_in (struct task_struct *task) +{ +JUMP_LABEL (&perf_task_events, have_events); +return ; + +have_events: +__perf_event_task_sched_in (task); +}
OK, so I hate the JUMP_LABEL() interface for it means we have to keep writing these silly stubs.
How about something like:
#define COND_STMT(key, stmt) \
do { \
__label__ jl_enabled; \
JUMP_LABEL(key, jl_enabled); \
if (0) { \
jl_enabled: \
stmt; \
} \
} while (0)Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks
Quoted 12 lines · Peter Zijlstra
Review
Quoted 8 lines · Peter Zijlstra
(quoted diff) +8
-0+static inline void perf_event_task_sched_in(struct task_struct *task) +{ + JUMP_LABEL(&perf_task_events, have_events); + return; + +have_events: + __perf_event_task_sched_in(task); +}OK, so I hate the
JUMP_LABEL()interface for it means we have to keep writing these silly stubs.
The below seems to actually compile too, awesome! ;-)
---
include/linux/jump_label.h +10 -0
@@ -61,4 +61,14 @@ static inline int jump_label_text_reserv#endif +#define COND_STMT (key, stmt) \ +do { \ + __label__ jl_enabled; \ + JUMP_LABEL(key, jl_enabled); \ + if (0) { \ +jl_enabled: \ + stmt; \ + } \ +} while (0) +#endif
include/linux/perf_event.h +2 -10
@@ -901,21 +901,13 @@ extern atomic_t perf_task_events;static inline void perf_event_task_sched_in (struct task_struct *task) { -JUMP_LABEL (&perf_task_events, have_events); -return ; - -have_events: -__perf_event_task_sched_in (task); +COND_STMT (&perf_task_events,__perf_event_task_sched_in (task)); }static inline void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next) { -JUMP_LABEL (&perf_task_events, have_events); -return ; - -have_events: -__perf_event_task_sched_out (task, next); +COND_STMT (&perf_task_events,__perf_event_task_sched_out (task, next)); }extern int perf_event_init_task (struct task_struct *child);
[tip:perf/core] perf: Fix task refcount bugs
In reply to Peter ZijlstraCommit-ID: e7d0bc047548d76feee6b23f7d3d9da927189a50 Gitweb: http://git.kernel.org/tip/e7d0bc047548d76feee6b23f7d3d9da927189a50 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 16:54:51 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:52 +0200
perf: Fix task refcount bugs
Currently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.
The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.
LKML-Reference: <[email protected]>
--- kernel/perf_event.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-)
kernel/perf_event.c +4 -3
@@ -2190,11 +2190,9 @@ retry: } } -put_task_struct (task);return ctx; errout: -put_task_struct (task);return ERR_PTR (err); }
@@ -5602,7 +5600,7 @@ SYSCALL_DEFINE5(perf_event_open, ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_group_fd; + goto err_task; }/*
@@ -5698,6 +5696,9 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_task: +if (task) +put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed);free_event (event);
[tip:perf/core] perf: Find task before event alloc
In reply to Peter ZijlstraCommit-ID: c6be5a5cb62592d9d661899a2aa78236eb00ffa5 Gitweb: http://git.kernel.org/tip/c6be5a5cb62592d9d661899a2aa78236eb00ffa5 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 16:59:46 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:54 +0200
perf: Find task before event alloc
So that we can pass the task pointer to the event allocation, so that we can use task associated data during event initialization.
LKML-Reference: <[email protected]>
--- kernel/perf_event.c | 23 ++++++++++++----------- 1 files changed, 12 insertions(+), 11 deletions(-)
kernel/perf_event.c +12 -11
@@ -5551,10 +5551,18 @@ SYSCALL_DEFINE5(perf_event_open, group_leader =NULL ; } +if (pid !=-1 ) { + task =find_lively_task_by_vpid (pid); +if (IS_ERR (task)) { + err =PTR_ERR (task); + goto err_group_fd; + } + } + event =perf_event_alloc (&attr, cpu, group_leader,NULL ,NULL );if (IS_ERR (event)) { err =PTR_ERR (event); - goto err_fd; + goto err_task; }/*
@@ -5586,21 +5594,13 @@ SYSCALL_DEFINE5(perf_event_open, } } -if (pid !=-1 ) { - task =find_lively_task_by_vpid (pid); -if (IS_ERR (task)) { - err =PTR_ERR (task); - goto err_group_fd; - } - } -/* * Get the target context (task or percpu): */ ctx =find_get_context (pmu, task, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_task; + goto err_alloc; }/*
@@ -5696,12 +5696,13 @@ SYSCALL_DEFINE5(perf_event_open, err_context:put_ctx (ctx); +err_alloc: +free_event (event); err_task:if (task)put_task_struct (task); err_group_fd:fput_light (group_file, fput_needed); -free_event (event); err_fd:put_unused_fd (event_fd);return err;
[tip:perf/core] perf, hw_breakpoint: Fix crash in hw_breakpoint creation
In reply to Peter ZijlstraCommit-ID: d580ff8699e8811a9af37e9de4dea375401bdeec Gitweb: http://git.kernel.org/tip/d580ff8699e8811a9af37e9de4dea375401bdeec Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 17:43:23 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:55 +0200
perf, hw_breakpoint: Fix crash in hw_breakpoint creation
hw_breakpoint creation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.
With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.
This makes hw_breakpoints unhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in the event::hw bits for now...
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 7 +++++++ kernel/hw_breakpoint.c | 8 ++++---- kernel/perf_event.c | 23 ++++++++++++++++++----- 3 files changed, 29 insertions(+), 9 deletions(-)
include/linux/perf_event.h +7 -0
@@ -536,6 +536,12 @@ struct hw_perf_event {struct {/* breakpoint */ struct arch_hw_breakpoint info;struct list_head bp_list; +/* +* Crufty hack to avoid the chicken and egg +* problem hw_breakpoint has with context +* creation and event initalization. +*/ +struct task_struct *bp_target; };#endif };
@@ -693,6 +699,7 @@ struct swevent_hlist {#define PERF_ATTACH_CONTEXT 0x01#define PERF_ATTACH_GROUP 0x02 +#define PERF_ATTACH_TASK 0x04/** * struct perf_event - performance event kernel representation:
kernel/hw_breakpoint.c +4 -4
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)*/ static int task_bp_pinned (struct perf_event *bp,enum bp_type_idx type) { -struct perf_event_context *ctx = bp->ctx; +struct task_struct *tsk = bp->hw.bp_target;struct perf_event *iter;int count =0 ;list_for_each_entry (iter, &bp_task_head, hw.bp_list) { -if (iter->ctx == ctx &&find_slot_idx (iter) == type) +if (iter->hw.bp_target == tsk &&find_slot_idx (iter) == type) count +=hw_breakpoint_weight (iter); }
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp, enum bp_type_idx type) {int cpu = bp->cpu; -struct task_struct *tsk = bp->ctx->task; +struct task_struct *tsk = bp->hw.bp_target;if (cpu >=0 ) { slots->pinned =per_cpu (nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type, int weight) {int cpu = bp->cpu; -struct task_struct *tsk = bp->ctx->task; +struct task_struct *tsk = bp->hw.bp_target;/* Pinned counter cpu profiling */ if (!tsk) {
kernel/perf_event.c +18 -5
@@ -5255,9 +5255,10 @@ unlock:*/ static struct perf_event *perf_event_alloc (struct perf_event_attr *attr,int cpu, -struct perf_event *group_leader, -struct perf_event *parent_event, -perf_overflow_handler_t overflow_handler) +struct task_struct *task, +struct perf_event *group_leader, +struct perf_event *parent_event, +perf_overflow_handler_t overflow_handler) {struct pmu *pmu;struct perf_event *event;
@@ -5299,6 +5300,17 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, event->state =PERF_EVENT_STATE_INACTIVE ; +if (task) { + event->attach_state =PERF_ATTACH_TASK ; +#ifdef CONFIG_HAVE_HW_BREAKPOINT +/* +* hw_breakpoint is a bit difficult here.. +*/ +if (attr->type ==PERF_TYPE_BREAKPOINT ) + event->hw.bp_target = task; +#endif + } +if (!overflow_handler && parent_event) overflow_handler = parent_event->overflow_handler;
@@ -5559,7 +5571,7 @@ SYSCALL_DEFINE5(perf_event_open, } } - event =perf_event_alloc (&attr, cpu, group_leader,NULL ,NULL ); + event =perf_event_alloc (&attr, cpu, task, group_leader,NULL ,NULL );if (IS_ERR (event)) { err =PTR_ERR (event); goto err_task;
@@ -5728,7 +5740,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,* Get the target context (task or percpu): */ - event =perf_event_alloc (attr, cpu,NULL ,NULL , overflow_handler); + event =perf_event_alloc (attr, cpu, task,NULL ,NULL , overflow_handler);if (IS_ERR (event)) { err =PTR_ERR (event); goto err;
@@ -5996,6 +6008,7 @@ inherit_event(struct perf_event *parent_event, child_event =perf_event_alloc (&parent_event->attr, parent_event->cpu, + child, group_leader, parent_event,NULL );if (IS_ERR (child_event))
[tip:perf/core] jump_label: Use more consistent naming
In reply to Peter ZijlstraCommit-ID: 3b6e901f839f42afb40f614418df82c08b01320a Gitweb: http://git.kernel.org/tip/3b6e901f839f42afb40f614418df82c08b01320a Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 21:10:38 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:56 +0200
jump_label: Use more consistent naming
Now that there's still only a few users around, rename things to make them more consistent.
LKML-Reference: <[email protected]>
--- include/linux/jump_label.h | 8 ++++---- kernel/tracepoint.c | 6 +++--- lib/dynamic_debug.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/jump_label.h +4 -4
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned long key, enum jump_label_type type);extern void jump_label_apply_nops (struct module *mod);extern int jump_label_text_reserved (void *start,void *end); -#define enable_jump_label (key) \ +#define jump_label_enable (key) \ jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE); -#define disable_jump_label (key) \ +#define jump_label_disable (key) \ jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);#else
@@ -39,12 +39,12 @@ do { \ goto label; \ } while (0) -#define enable_jump_label (cond_var) \ +#define jump_label_enable (cond_var) \ do { \ *(cond_var) = 1; \ } while (0) -#define disable_jump_label (cond_var) \ +#define jump_label_disable (cond_var) \ do { \ *(cond_var) = 0; \ } while (0)
kernel/tracepoint.c +3 -3
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,*/ rcu_assign_pointer (elem->funcs, (*entry)->funcs);if (!elem->state && active) { -enable_jump_label (&elem->state); +jump_label_enable (&elem->state); elem->state = active; }else if (elem->state && !active) { -disable_jump_label (&elem->state); +jump_label_disable (&elem->state); elem->state = active; } }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tracepoint *elem) elem->unregfunc ();if (elem->state) { -disable_jump_label (&elem->state); +jump_label_disable (&elem->state); elem->state =0 ; }rcu_assign_pointer (elem->funcs,NULL );
lib/dynamic_debug.c +2 -2
@@ -142,9 +142,9 @@ static void ddebug_change(const struct ddebug_query *query, dt->num_enabled++; dp->flags = newflags;if (newflags) { -enable_jump_label (&dp->enabled); +jump_label_enable (&dp->enabled); }else { -disable_jump_label (&dp->enabled); +jump_label_disable (&dp->enabled); }if (verbose) printk(KERN_INFO
[tip:perf/core] jump_label: Add atomic_t interface
In reply to Peter ZijlstraCommit-ID: 8b92538d84e50062560ba33adbaed7887b6e4a42 Gitweb: http://git.kernel.org/tip/8b92538d84e50062560ba33adbaed7887b6e4a42 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 21:39:02 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:57 +0200
jump_label: Add atomic_t interface
Add an interface to allow usage of jump_labels with atomic counters.
LKML-Reference: <[email protected]>
--- include/linux/jump_label_ref.h | 44 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 44 insertions(+), 0 deletions(-)
include/linux/jump_label_ref.h new file +44 -0
@@ -0,0 +1,44 @@ +#ifndef _LINUX_JUMP_LABEL_REF_H +#define _LINUX_JUMP_LABEL_REF_H + +#include <linux/jump_label.h> +#include <asm/atomic.h> + +#ifdef HAVE_JUMP_LABEL + +static inline void jump_label_inc (atomic_t *key) +{ +if (atomic_add_return (1 , key) ==1 ) +jump_label_enable (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +if (atomic_dec_and_test (key)) +jump_label_disable (key); +} + +#else /* !HAVE_JUMP_LABEL */ + +static inline void jump_label_inc (atomic_t *key) +{ +atomic_inc (key); +} + +static inline void jump_label_dec (atomic_t *key) +{ +atomic_dec (key); +} + +#undef JUMP_LABEL +#define JUMP_LABEL (key, label) \ +do { \ + if (unlikely(__builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(key), atomic_t *), \ + atomic_read((atomic_t *)(key)), *(key)))) \ + goto label; \ +} while (0) + +#endif /* HAVE_JUMP_LABEL */ + +#endif /* _LINUX_JUMP_LABEL_REF_H */
[tip:perf/core] perf: Use jump_labels to optimize the scheduler hooks
In reply to Peter ZijlstraCommit-ID: 82cd6def9806dcb6a325fb6abbc1d61388a15f6a Gitweb: http://git.kernel.org/tip/82cd6def9806dcb6a325fb6abbc1d61388a15f6a Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 14 Oct 2010 17:57:23 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 18 Oct 2010 19:58:58 +0200
perf: Use jump_labels to optimize the scheduler hooks
Trades a call + conditional + ret for an unconditional jmp.
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 27 +++++++++++++++++++++++++-- kernel/perf_event.c | 24 +++++++++--------------- 2 files changed, 34 insertions(+), 17 deletions(-)
include/linux/perf_event.h +25 -2
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {#include <linux/ftrace.h> #include <linux/cpu.h> #include <linux/irq_work.h> +#include <linux/jump_label_ref.h> #include <asm/atomic.h> #include <asm/local.h>
@@ -895,8 +896,30 @@ extern void perf_pmu_unregister(struct pmu *pmu);extern int perf_num_counters (void );extern const char *perf_pmu_name (void ); -extern void perf_event_task_sched_in (struct task_struct *task); -extern void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next); +extern void __perf_event_task_sched_in (struct task_struct *task); +extern void __perf_event_task_sched_out (struct task_struct *task,struct task_struct *next); + +extern atomic_t perf_task_events; + +static inline void perf_event_task_sched_in (struct task_struct *task) +{ +JUMP_LABEL (&perf_task_events, have_events); +return ; + +have_events: +__perf_event_task_sched_in (task); +} + +static inline +void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next) +{ +JUMP_LABEL (&perf_task_events, have_events); +return ; + +have_events: +__perf_event_task_sched_out (task, next); +} +extern int perf_event_init_task (struct task_struct *child);extern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);
kernel/perf_event.c +9 -15
@@ -34,7 +34,7 @@#include <asm/irq_regs.h> -static atomic_t nr_events__read_mostly ; +atomic_t perf_task_events__read_mostly ;static atomic_t nr_mmap_events__read_mostly ;static atomic_t nr_comm_events__read_mostly ;static atomic_t nr_task_events__read_mostly ;
@@ -1311,8 +1311,8 @@ void perf_event_context_sched_out(struct task_struct *task, int ctxn,* accessing the event control register. If a NMI hits, then it will * not restart the event. */ -void perf_event_task_sched_out (struct task_struct *task, -struct task_struct *next) +void __perf_event_task_sched_out (struct task_struct *task, +struct task_struct *next) {int ctxn;
@@ -1340,14 +1340,6 @@ static void task_ctx_sched_out(struct perf_event_context *ctx,/* * Called with IRQs disabled */ -static void __perf_event_task_sched_out (struct perf_event_context *ctx) -{ -task_ctx_sched_out (ctx,EVENT_ALL ); -} - -/* -* Called with IRQs disabled -*/ static void cpu_ctx_sched_out (struct perf_cpu_context *cpuctx,enum event_type_t event_type) {
@@ -1494,7 +1486,7 @@ void perf_event_context_sched_in(struct perf_event_context *ctx)* accessing the event control register. If a NMI hits, then it will * keep the event running. */ -void perf_event_task_sched_in (struct task_struct *task) +void __perf_event_task_sched_in (struct task_struct *task) {struct perf_event_context *ctx;int ctxn;
@@ -2216,7 +2208,8 @@ static void free_event(struct perf_event *event)irq_work_sync (&event->pending);if (!event->parent) { -atomic_dec (&nr_events); +if (event->attach_state &PERF_ATTACH_TASK ) +jump_label_dec (&perf_task_events);if (event->attr.mmap || event->attr.mmap_data)atomic_dec (&nr_mmap_events);if (event->attr.comm)
@@ -5354,7 +5347,8 @@ done: event->pmu = pmu;if (!event->parent) { -atomic_inc (&nr_events); +if (event->attach_state &PERF_ATTACH_TASK ) +jump_label_inc (&perf_task_events);if (event->attr.mmap || event->attr.mmap_data)atomic_inc (&nr_mmap_events);if (event->attr.comm)
@@ -5849,7 +5843,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)* our context. */ child_ctx = child->perf_event_ctxp[ctxn]; -__perf_event_task_sched_out (child_ctx); +task_ctx_sched_out (child_ctx,EVENT_ALL );/* * Take the context lock here so that if find_get_context is