perf pmu interface changes -v4
Subsystems (11)
RFC PATCH 12/19 perf: Remove the swevent hash-table from the cpu context
In reply to Peter ZijlstraSeparate the swevent hash-table from the cpu_context bits in preparation for per pmu cpu contexts.
This keeps the swevent hash a global entity.
--- include/linux/perf_event.h | 6 -- kernel/perf_event.c | 104 +++++++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 52 deletions(-)
include/linux/perf_event.h +0 -6
@@ -866,12 +866,6 @@ struct perf_cpu_context {struct perf_event_context *task_ctx;int active_oncpu;int exclusive; -struct swevent_hlist *swevent_hlist; -struct mutex hlist_mutex; -int hlist_refcount; - -/* Recursion avoidance in each contexts */ -int recursion[PERF_NR_CONTEXTS ]; };struct perf_output_handle {
kernel/perf_event.c +58 -46
243 lines not on this page · raw message
RFC PATCH 18/19 perf: Provide a separate task context for swevents
In reply to Peter ZijlstraSince software events are always schedulable, mixing them up with hardware events (who are not) can lead to funny scheduling oddities.
Giving them their own context solves this.
--- include/linux/sched.h | 1 + kernel/hw_breakpoint.c | 2 ++ kernel/perf_event.c | 8 ++++++++ 3 files changed, 11 insertions(+)
include/linux/sched.h +1 -0
@@ -1164,6 +1164,7 @@ struct rcu_node;enum perf_event_task_context { perf_invalid_context =-1 , perf_hw_context =0 , + perf_sw_context, perf_nr_task_contexts, };
kernel/perf_event.c +8 -0
@@ -4691,6 +4691,8 @@ static int perf_swevent_init(struct perf }static struct pmu perf_swevent = { + .task_ctx_nr = perf_sw_context, + .event_init = perf_swevent_init, .add = perf_swevent_add, .del = perf_swevent_del,
@@ -4782,6 +4784,8 @@ static int perf_tp_event_init(struct per }static struct pmu perf_tracepoint = { + .task_ctx_nr = perf_sw_context, + .event_init = perf_tp_event_init, .add = perf_trace_add, .del = perf_trace_del,
@@ -4970,6 +4974,8 @@ static int cpu_clock_event_init(struct p }static struct pmu perf_cpu_clock = { + .task_ctx_nr = perf_sw_context, + .event_init = cpu_clock_event_init, .add = cpu_clock_event_add, .del = cpu_clock_event_del,
@@ -5045,6 +5051,8 @@ static int task_clock_event_init(struct }static struct pmu perf_task_clock = { + .task_ctx_nr = perf_sw_context, + .event_init = task_clock_event_init, .add = task_clock_event_add, .del = task_clock_event_del,
kernel/hw_breakpoint.c +2 -0
@@ -610,6 +610,8 @@ static void hw_breakpoint_stop(struct pe }static struct pmu perf_breakpoint = { + .task_ctx_nr = perf_sw_context,/* could eventually get its own */ + .event_init = hw_breakpoint_event_init, .add = hw_breakpoint_add, .del = hw_breakpoint_del,
RFC PATCH 8/19 perf: Shrink hw_perf_event
In reply to Peter ZijlstraUse hw_perf_event::period_left instead of hw_perf_event::remaning and win back 8 bytes.
--- include/linux/perf_event.h | 1 - kernel/perf_event.c | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-)
include/linux/perf_event.h +0 -1
@@ -529,7 +529,6 @@ struct hw_perf_event {int last_cpu; };struct {/* software */ -s64 remaining;struct hrtimer hrtimer; };#ifdef CONFIG_HAVE_HW_BREAKPOINT
kernel/perf_event.c +6 -7
@@ -4590,14 +4590,13 @@ static void perf_swevent_start_hrtimer(shrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); hwc->hrtimer.function = perf_swevent_hrtimer;if (hwc->sample_period) { -u64 period; +s64 period =local64_read (&hwc->period_left); -if (hwc->remaining) { -if (hwc->remaining <0 ) +if (period) { +if (period <0 ) period =10000 ; -else - period = hwc->remaining; - hwc->remaining =0 ; + +local64_set (&hwc->period_left,0 ); }else { period =max_t (u64,10000 , hwc->sample_period); }
@@ -4613,7 +4612,7 @@ static void perf_swevent_cancel_hrtimer(if (hwc->sample_period) {ktime_t remaining =hrtimer_get_remaining (&hwc->hrtimer); - hwc->remaining =ktime_to_ns (remaining); +local64_set (&hwc->period_left,ktime_to_ns (remaining));hrtimer_cancel (&hwc->hrtimer); }
RFC PATCH 1/19 perf: Fix CPU hotplug
In reply to Peter ZijlstraSince we have UP_PREPARE, we should also have UP_CANCELED.
--- kernel/perf_event.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/perf_event.c +3 -3
@@ -5743,15 +5743,15 @@ perf_cpu_notify(struct notifier_block *s {unsigned int cpu = (long )hcpu; -switch (action) { +switch (action & ~CPU_TASKS_FROZEN ) {case CPU_UP_PREPARE : -case CPU_UP_PREPARE_FROZEN : +case CPU_DOWN_FAILED :perf_event_init_cpu (cpu);break ; +case CPU_UP_CANCELED :case CPU_DOWN_PREPARE : -case CPU_DOWN_PREPARE_FROZEN :perf_event_exit_cpu (cpu);break ;
RFC PATCH 15/19 perf: Move some code around
In reply to Peter ZijlstraMove all inherit code near each other.
--- kernel/perf_event.c | 200 ++++++++++++++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 100 deletions(-)
kernel/perf_event.c +100 -100
@@ -5537,106 +5537,6 @@ perf_event_create_kernel_counter(struct }EXPORT_SYMBOL_GPL (perf_event_create_kernel_counter); -/* -* inherit a event from parent task to child task: -*/ -static struct perf_event * -inherit_event (struct perf_event *parent_event, -struct task_struct *parent, -struct perf_event_context *parent_ctx, -struct task_struct *child, -struct perf_event *group_leader, -struct perf_event_context *child_ctx) -{ -struct perf_event *child_event; - -/* -* Instead of creating recursive hierarchies of events, -* we link inherited events back to the original parent, -* which has a filp for sure, which we use as the reference -* count: -*/ -if (parent_event->parent) - parent_event = parent_event->parent; - - child_event =perf_event_alloc (&parent_event->attr, - parent_event->cpu, - group_leader, parent_event, -NULL ); -if (IS_ERR (child_event)) -return child_event; -get_ctx (child_ctx); - -/* -* Make the child state follow the state of the parent event, -* not its attr.disabled bit. We hold the parent's mutex, -* so we won't race with perf_event_{en, dis}able_family. -*/ -if (parent_event->state >=PERF_EVENT_STATE_INACTIVE ) - child_event->state =PERF_EVENT_STATE_INACTIVE ; -else - child_event->state =PERF_EVENT_STATE_OFF ; - -if (parent_event->attr.freq) { -u64 sample_period = parent_event->hw.sample_period; -struct hw_perf_event *hwc = &child_event->hw; - - hwc->sample_period = sample_period; - hwc->last_period = sample_period; - -local64_set (&hwc->period_left, sample_period); - } - - child_event->ctx = child_ctx; - child_event->overflow_handler = parent_event->overflow_handler; - -/* -* Link it up in the child's context: -*/ -add_event_to_ctx (child_event, child_ctx); - -/* -* Get a reference to the parent filp - we will fput it -* when the child event exits. This is safe to do because -* we are in the parent and we know that the filp still -* exists and has a nonzero count: -*/ -atomic_long_inc (&parent_event->filp->f_count); - -/* -* Link this into the parent event's child list -*/ -WARN_ON_ONCE (parent_event->ctx->parent_ctx); -mutex_lock (&parent_event->child_mutex); -list_add_tail (&child_event->child_list, &parent_event->child_list); -mutex_unlock (&parent_event->child_mutex); - -return child_event; -} - -static int inherit_group (struct perf_event *parent_event, -struct task_struct *parent, -struct perf_event_context *parent_ctx, -struct task_struct *child, -struct perf_event_context *child_ctx) -{ -struct perf_event *leader; -struct perf_event *sub; -struct perf_event *child_ctr; - - leader =inherit_event (parent_event, parent, parent_ctx, - child,NULL , child_ctx); -if (IS_ERR (leader)) -return PTR_ERR (leader); -list_for_each_entry (sub, &parent_event->sibling_list, group_entry) { - child_ctr =inherit_event (sub, parent, parent_ctx, - child, leader, child_ctx); -if (IS_ERR (child_ctr)) -return PTR_ERR (child_ctr); - } -return 0 ; -} -static void sync_child_event (struct perf_event *child_event,struct task_struct *child) {
@@ -5825,6 +5725,106 @@ void perf_event_free_task(struct task_stput_ctx (ctx); } +/* +* inherit a event from parent task to child task: +*/ +static struct perf_event * +inherit_event (struct perf_event *parent_event, +struct task_struct *parent, +struct perf_event_context *parent_ctx, +struct task_struct *child, +struct perf_event *group_leader, +struct perf_event_context *child_ctx) +{ +struct perf_event *child_event; + +/* +* Instead of creating recursive hierarchies of events, +* we link inherited events back to the original parent, +* which has a filp for sure, which we use as the reference +* count: +*/ +if (parent_event->parent) + parent_event = parent_event->parent; + + child_event =perf_event_alloc (&parent_event->attr, + parent_event->cpu, + group_leader, parent_event, +NULL ); +if (IS_ERR (child_event)) +return child_event; +get_ctx (child_ctx); + +/* +* Make the child state follow the state of the parent event, +* not its attr.disabled bit. We hold the parent's mutex, +* so we won't race with perf_event_{en, dis}able_family. +*/ +if (parent_event->state >=PERF_EVENT_STATE_INACTIVE ) + child_event->state =PERF_EVENT_STATE_INACTIVE ; +else + child_event->state =PERF_EVENT_STATE_OFF ; + +if (parent_event->attr.freq) { +u64 sample_period = parent_event->hw.sample_period; +struct hw_perf_event *hwc = &child_event->hw; + + hwc->sample_period = sample_period; + hwc->last_period = sample_period; + +local64_set (&hwc->period_left, sample_period); + } + + child_event->ctx = child_ctx; + child_event->overflow_handler = parent_event->overflow_handler; + +/* +* Link it up in the child's context: +*/ +add_event_to_ctx (child_event, child_ctx); + +/* +* Get a reference to the parent filp - we will fput it +* when the child event exits. This is safe to do because +* we are in the parent and we know that the filp still +* exists and has a nonzero count: +*/ +atomic_long_inc (&parent_event->filp->f_count); + +/* +* Link this into the parent event's child list +*/ +WARN_ON_ONCE (parent_event->ctx->parent_ctx); +mutex_lock (&parent_event->child_mutex); +list_add_tail (&child_event->child_list, &parent_event->child_list); +mutex_unlock (&parent_event->child_mutex); + +return child_event; +} + +static int inherit_group (struct perf_event *parent_event, +struct task_struct *parent, +struct perf_event_context *parent_ctx, +struct task_struct *child, +struct perf_event_context *child_ctx) +{ +struct perf_event *leader; +struct perf_event *sub; +struct perf_event *child_ctr; + + leader =inherit_event (parent_event, parent, parent_ctx, + child,NULL , child_ctx); +if (IS_ERR (leader)) +return PTR_ERR (leader); +list_for_each_entry (sub, &parent_event->sibling_list, group_entry) { + child_ctr =inherit_event (sub, parent, parent_ctx, + child, leader, child_ctx); +if (IS_ERR (child_ctr)) +return PTR_ERR (child_ctr); + } +return 0 ; +} + static int inherit_task_group(struct perf_event *event, struct task_struct *parent, struct perf_event_context *parent_ctx,
RFC PATCH 19/19 perf: Optimize context ops
In reply to Peter ZijlstraAssuming we don't mix events of different pmus onto a single context (with the exeption of software events inside a hardware group) we can now assume that all events on a particular context belong to the same pmu, hence we can disable the pmu for the entire context operations.
This reduces the amount of hardware writes.
The exception for swevents comes from the fact that the sw pmu disable is a nop.
--- kernel/perf_event.c | 6 ++++++ 1 file changed, 6 insertions(+)
kernel/perf_event.c +6 -0
@@ -1046,6 +1046,7 @@ static void ctx_sched_out(struct perf_evstruct perf_event *event;raw_spin_lock (&ctx->lock); +perf_pmu_disable (ctx->pmu); ctx->is_active =0 ;if (likely (!ctx->nr_events)) goto out;
@@ -1064,6 +1065,7 @@ static void ctx_sched_out(struct perf_evgroup_sched_out (event, cpuctx, ctx); } out: +perf_pmu_enable (ctx->pmu);raw_spin_unlock (&ctx->lock); }
@@ -1386,6 +1388,7 @@ void perf_event_context_sched_in(structif (cpuctx->task_ctx == ctx)return ; +perf_pmu_disable (ctx->pmu);/* * We want to keep the following priority order: * cpu pinned (that don't need to move), task pinned,
@@ -1398,6 +1401,7 @@ void perf_event_context_sched_in(structctx_sched_in (ctx, cpuctx,EVENT_FLEXIBLE ); cpuctx->task_ctx = ctx; +perf_pmu_enable (ctx->pmu); }/*
@@ -1602,6 +1606,7 @@ static enum hrtimer_restart perf_event_c rotate =1 ; } +perf_pmu_disable (cpuctx->ctx.pmu);perf_ctx_adjust_freq (&cpuctx->ctx, cpuctx->timer_interval);if (ctx)perf_ctx_adjust_freq (ctx, cpuctx->timer_interval);
@@ -1622,6 +1627,7 @@ static enum hrtimer_restart perf_event_ctask_ctx_sched_in (ctx,EVENT_FLEXIBLE ); done: +perf_pmu_enable (cpuctx->ctx.pmu);hrtimer_forward_now (timer,ns_to_ktime (cpuctx->timer_interval));return restart;
RFC PATCH 5/19 perf: Reduce perf_disable() usage
In reply to Peter ZijlstraSince the current perf_disable() usage is only an optimization, remove it for now. This eases the removal of the weak hw_perf_enable interface.
--- arch/arm/kernel/perf_event.c | 3 ++ arch/powerpc/kernel/perf_event.c | 3 ++ arch/powerpc/kernel/perf_event_fsl_emb.c | 8 +++++- arch/sh/kernel/perf_event.c | 11 ++++++--- arch/sparc/kernel/perf_event.c | 3 ++ arch/x86/kernel/cpu/perf_event.c | 22 +++++++++++------- include/linux/perf_event.h | 20 ++++++++-------- kernel/perf_event.c | 37 ------------------------------- 8 files changed, 48 insertions(+), 59 deletions(-)
include/linux/perf_event.h +10 -10
@@ -564,26 +564,26 @@ struct pmu {struct list_head entry;/* -* Should return -ENOENT when the @event doesn't match this pmu +* Should return -ENOENT when the @event doesn't match this PMU. */ int (*event_init) (struct perf_event *event); -int (*enable) (struct perf_event *event); +int (*enable) (struct perf_event *event);void (*disable) (struct perf_event *event); -int (*start) (struct perf_event *event); +int (*start) (struct perf_event *event);void (*stop) (struct perf_event *event);void (*read) (struct perf_event *event);void (*unthrottle) (struct perf_event *event);/* -* Group events scheduling is treated as a transaction, add group -* events as a whole and perform one schedulability test. If the test -* fails, roll back the whole group +* Group events scheduling is treated as a transaction, add +* group events as a whole and perform one schedulability test. +* If the test fails, roll back the whole group */ /* -* Start the transaction, after this ->enable() doesn't need -* to do schedulability tests. +* Start the transaction, after this ->enable() doesn't need to +* do schedulability tests. */ void (*start_txn) (struct pmu *pmu);/*
@@ -594,8 +594,8 @@ struct pmu {*/ int (*commit_txn) (struct pmu *pmu);/* -* Will cancel the transaction, assumes ->disable() is called for -* each successfull ->enable() during the transaction. +* Will cancel the transaction, assumes ->disable() is called +* for each successfull ->enable() during the transaction. */ void (*cancel_txn) (struct pmu *pmu); };
kernel/perf_event.c +1 -36
@@ -460,11 +460,6 @@ static void __perf_event_remove_from_conreturn ;raw_spin_lock (&ctx->lock); -/* -* Protect the list operation against NMI by disabling the -* events on a global level. -*/ -perf_disable ();event_sched_out (event, cpuctx, ctx);
@@ -480,7 +475,6 @@ static void __perf_event_remove_from_con perf_max_events - perf_reserved_percpu); } -perf_enable ();raw_spin_unlock (&ctx->lock); }
@@ -785,12 +779,6 @@ static void __perf_install_in_context(vo ctx->is_active =1 ;update_context_time (ctx); -/* -* Protect the list operation against NMI by disabling the -* events on a global level. NOP for non NMI based events. -*/ -perf_disable (); -add_event_to_ctx (event, ctx);if (event->cpu !=-1 && event->cpu !=smp_processor_id ())
@@ -832,8 +820,6 @@ static void __perf_install_in_context(vo cpuctx->max_pertask--; unlock: -perf_enable (); -raw_spin_unlock (&ctx->lock); }
@@ -954,12 +940,10 @@ static void __perf_event_enable(void *inif (!group_can_go_on (event, cpuctx,1 )) { err = -EEXIST ; }else { -perf_disable ();if (event == leader) err =group_sched_in (event, cpuctx, ctx);else err =event_sched_in (event, cpuctx, ctx); -perf_enable (); }if (err) {
@@ -1072,9 +1056,8 @@ static void ctx_sched_out(struct perf_ev goto out;update_context_time (ctx); -perf_disable ();if (!ctx->nr_active) - goto out_enable; + goto out;if (event_type &EVENT_PINNED ) {list_for_each_entry (event, &ctx->pinned_groups, group_entry)
@@ -1085,9 +1068,6 @@ static void ctx_sched_out(struct perf_evlist_for_each_entry (event, &ctx->flexible_groups, group_entry)group_sched_out (event, cpuctx, ctx); } - - out_enable: -perf_enable (); out:raw_spin_unlock (&ctx->lock); }
@@ -1346,8 +1326,6 @@ ctx_sched_in(struct perf_event_context * ctx->timestamp =perf_clock (); -perf_disable (); -/* * First go through the list and put on any pinned groups * in order to give them the best chance of going on.
@@ -1359,7 +1337,6 @@ ctx_sched_in(struct perf_event_context *if (event_type &EVENT_FLEXIBLE )ctx_flexible_sched_in (ctx, cpuctx); -perf_enable (); out:raw_spin_unlock (&ctx->lock); }
@@ -1407,8 +1384,6 @@ void perf_event_task_sched_in(struct tasif (cpuctx->task_ctx == ctx)return ; -perf_disable (); -/* * We want to keep the following priority order: * cpu pinned (that don't need to move), task pinned,
@@ -1421,8 +1396,6 @@ void perf_event_task_sched_in(struct tasctx_sched_in (ctx, cpuctx,EVENT_FLEXIBLE ); cpuctx->task_ctx = ctx; - -perf_enable (); }#define MAX_INTERRUPTS (~0ULL)
@@ -1537,11 +1510,9 @@ static void perf_adjust_period(struct pe hwc->sample_period = sample_period;if (local64_read (&hwc->period_left) >8 *sample_period) { -perf_disable ();perf_event_stop (event);local64_set (&hwc->period_left,0 );perf_event_start (event); -perf_enable (); } }
@@ -1570,15 +1541,12 @@ static void perf_ctx_adjust_freq(struct*/ if (interrupts ==MAX_INTERRUPTS ) {perf_log_throttle (event,1 ); -perf_disable (); event->pmu->unthrottle (event); -perf_enable (); }if (!event->attr.freq || !event->attr.sample_freq)continue ; -perf_disable (); event->pmu->read (event); now =local64_read (&event->count); delta = now - hwc->freq_count_stamp;
@@ -1586,7 +1554,6 @@ static void perf_ctx_adjust_freq(structif (delta >0 )perf_adjust_period (event,TICK_NSEC , delta); -perf_enable (); }raw_spin_unlock (&ctx->lock); }
@@ -1629,7 +1596,6 @@ void perf_event_task_tick(struct task_stif (!rotate)return ; -perf_disable ();cpu_ctx_sched_out (cpuctx,EVENT_FLEXIBLE );if (ctx)task_ctx_sched_out (ctx,EVENT_FLEXIBLE );
@@ -1641,7 +1607,6 @@ void perf_event_task_tick(struct task_stcpu_ctx_sched_in (cpuctx,EVENT_FLEXIBLE );if (ctx)task_ctx_sched_in (curr,EVENT_FLEXIBLE ); -perf_enable (); } static int event_enable_on_exec(struct perf_event *event,
arch/powerpc/kernel/perf_event_fsl_emb.c +6 -2
@@ -262,7 +262,7 @@ static int collect_events(struct perf_evreturn n; } -/* perf must be disabled, context locked on entry */ +/* context locked on entry */ static int fsl_emb_pmu_enable (struct perf_event *event) {struct cpu_hw_events *cpuhw;
@@ -271,6 +271,7 @@ static int fsl_emb_pmu_enable(struct peru64 val;int i; +perf_disable (); cpuhw = &get_cpu_var (cpu_hw_events);if (event->hw.config &FSL_EMB_EVENT_RESTRICTED )
@@ -310,15 +311,17 @@ static int fsl_emb_pmu_enable(struct per ret =0 ; out:put_cpu_var (cpu_hw_events); +perf_enable ();return ret; } -/* perf must be disabled, context locked on entry */ +/* context locked on entry */ static void fsl_emb_pmu_disable (struct perf_event *event) {struct cpu_hw_events *cpuhw;int i = event->hw.idx; +perf_disable ();if (i <0 ) goto out;
@@ -346,6 +349,7 @@ static void fsl_emb_pmu_disable(struct p cpuhw->n_events--; out: +perf_enable ();put_cpu_var (cpu_hw_events); }
arch/x86/kernel/cpu/perf_event.c +14 -8
@@ -969,10 +969,11 @@ static int x86_pmu_enable(struct perf_ev hwc = &event->hw; +perf_disable (); n0 = cpuc->n_events; - n =collect_events (cpuc, event, false); -if (n <0 ) -return n; + ret = n =collect_events (cpuc, event, false); +if (ret <0 ) + goto out;/* * If group events scheduling transaction was started,
@@ -980,23 +981,26 @@ static int x86_pmu_enable(struct perf_ev* at commit time(->commit_txn) as a whole */ if (cpuc->group_flag &PERF_EVENT_TXN ) - goto out; + goto done_collect; ret = x86_pmu.schedule_events (cpuc, n, assign);if (ret) -return ret; + goto out;/* * copy new assignment, now we know it is possible * will be used by hw_perf_enable() */ memcpy (cpuc->assign, assign, n*sizeof (int )); -out: +done_collect: cpuc->n_events = n; cpuc->n_added += n - n0; cpuc->n_txn += n - n0; -return 0 ; + ret =0 ; +out: +perf_enable (); +return ret; }static int x86_pmu_start (struct perf_event *event)
@@ -1399,6 +1403,7 @@ static void x86_pmu_start_txn(struct pmu {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +perf_disable (); cpuc->group_flag |=PERF_EVENT_TXN ; cpuc->n_txn =0 ; }
@@ -1418,6 +1423,7 @@ static void x86_pmu_cancel_txn(struct pm*/ cpuc->n_added -= cpuc->n_txn; cpuc->n_events -= cpuc->n_txn; +perf_enable (); }/*
@@ -1447,7 +1453,7 @@ static int x86_pmu_commit_txn(struct pmumemcpy (cpuc->assign, assign, n*sizeof (int )); cpuc->group_flag &= ~PERF_EVENT_TXN ; - +perf_enable ();return 0 ; }
arch/powerpc/kernel/perf_event.c +3 -0
@@ -861,6 +861,7 @@ void power_pmu_start_txn(struct pmu *pmu {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +perf_disable (); cpuhw->group_flag |=PERF_EVENT_TXN ; cpuhw->n_txn_start = cpuhw->n_events; }
@@ -875,6 +876,7 @@ void power_pmu_cancel_txn(struct pmu *pmstruct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ; +perf_enable (); }/*
@@ -901,6 +903,7 @@ int power_pmu_commit_txn(struct pmu *pmu cpuhw->event[i]->hw.config = cpuhw->events[i]; cpuhw->group_flag &= ~PERF_EVENT_TXN ; +perf_enable ();return 0 ; }
arch/sparc/kernel/perf_event.c +3 -0
@@ -1113,6 +1113,7 @@ static void sparc_pmu_start_txn(struct p {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); +perf_disable (); cpuhw->group_flag |=PERF_EVENT_TXN ; }
@@ -1126,6 +1127,7 @@ static void sparc_pmu_cancel_txn(structstruct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ; +perf_enable (); }/*
@@ -1149,6 +1151,7 @@ static int sparc_pmu_commit_txn(struct preturn -EAGAIN ; cpuc->group_flag &= ~PERF_EVENT_TXN ; +perf_enable ();return 0 ; }
arch/arm/kernel/perf_event.c +3 -0
@@ -277,6 +277,8 @@ armpmu_enable(struct perf_event *event)int idx;int err =0 ; +perf_disable (); +/* If we don't have a space for the counter then finish early. */ idx = armpmu->get_event_idx (cpuc, hwc);if (idx <0 ) {
@@ -303,6 +305,7 @@ armpmu_enable(struct perf_event *event)perf_event_update_userpage (event); out: +perf_enable ();return err; }
arch/sh/kernel/perf_event.c +8 -3
@@ -230,11 +230,14 @@ static int sh_pmu_enable(struct perf_evestruct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx; +int ret = -EAGAIN ; + +perf_disable ();if (test_and_set_bit (idx, cpuc->used_mask)) { idx =find_first_zero_bit (cpuc->used_mask, sh_pmu->num_events);if (idx == sh_pmu->num_events) -return -EAGAIN ; + goto out;set_bit (idx, cpuc->used_mask); hwc->idx = idx;
@@ -248,8 +251,10 @@ static int sh_pmu_enable(struct perf_eve sh_pmu->enable (hwc, idx);perf_event_update_userpage (event); - -return 0 ; + ret =0 ; +out: +perf_enable (); +return ret; }static void sh_pmu_read (struct perf_event *event)
RFC PATCH 13/19 perf: Per cpu-context rotation timer
In reply to Peter ZijlstraGive each cpu-context its own timer so that it is a self contained entity, this eases the way for per-pmu-per-cpu contexts as well as provides means of overloading the event rotation per pmu implementation as requested by Corey some time ago.
--- include/linux/perf_event.h | 10 +++++-- kernel/perf_event.c | 61 +++++++++++++++++++++++++++++++++------------ kernel/sched.c | 2 - 3 files changed, 52 insertions(+), 21 deletions(-)
include/linux/perf_event.h +7 -3
@@ -580,6 +580,11 @@ struct pmu {void (*pmu_disable) (struct pmu *pmu);/* optional */ /* +* Provide for means to modify the normal overcommit rotation scheme. +*/ +void (*pmu_rotate_start) (struct pmu *pmu);/* optional */ + +/* * Try and initialize the event for this PMU. * Should return -ENOENT when the @event doesn't match this PMU. */
@@ -861,6 +866,8 @@ struct perf_cpu_context {struct perf_event_context *task_ctx;int active_oncpu;int exclusive; +u64 timer_interval; +struct hrtimer timer; };struct perf_output_handle {
@@ -881,7 +888,6 @@ extern void perf_pmu_unregister(struct pextern void perf_event_task_sched_in (struct task_struct *task);extern void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next); -extern void perf_event_task_tick (struct task_struct *task);extern int perf_event_init_task (struct task_struct *child);extern void perf_event_exit_task (struct task_struct *child);extern void perf_event_free_task (struct task_struct *task);
@@ -1067,8 +1073,6 @@ perf_event_task_sched_in(struct task_strstatic inline void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next) { } -static inline void -perf_event_task_tick (struct task_struct *task) { }static inline int perf_event_init_task (struct task_struct *child) {return 0 ; }static inline void perf_event_exit_task (struct task_struct *child) { }static inline void perf_event_free_task (struct task_struct *task) { }
kernel/sched.c +0 -2
@@ -3441,8 +3441,6 @@ void scheduler_tick(void) curr->sched_class->task_tick (rq, curr,0 );raw_spin_unlock (&rq->lock); -perf_event_task_tick (curr); -#ifdef CONFIG_SMP rq->idle_at_tick =idle_cpu (cpu);trigger_load_balance (rq, cpu);
kernel/perf_event.c +45 -16
114 lines not on this page · raw message
RFC PATCH 11/19 perf: Separate find_get_context() from event initialization
In reply to Peter ZijlstraSeparate find_get_context() from the event allocation and initialization so that we may make find_get_context() depend on the event pmu in a later patch.
--- kernel/perf_event.c | 73 ++++++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 38 deletions(-)
kernel/perf_event.c +35 -38
@@ -827,6 +827,8 @@ perf_install_in_context(struct perf_even {struct task_struct *task = ctx->task; + event->ctx = ctx; +if (!task) {/* * Per cpu events are installed via an smp call and
@@ -5037,20 +5039,17 @@ struct pmu *perf_init_event(struct perf_* Allocate and initialize a event structure */ static struct perf_event * -perf_event_alloc (struct perf_event_attr *attr, -int cpu, -struct perf_event_context *ctx, +perf_event_alloc (struct perf_event_attr *attr,int cpu,struct perf_event *group_leader,struct perf_event *parent_event, -perf_overflow_handler_t overflow_handler, -gfp_t gfpflags) +perf_overflow_handler_t overflow_handler) {struct pmu *pmu;struct perf_event *event;struct hw_perf_event *hwc;long err; - event =kzalloc (sizeof (*event), gfpflags); + event =kzalloc (sizeof (*event),GFP_KERNEL );if (!event)return ERR_PTR (-ENOMEM );
@@ -5075,7 +5074,6 @@ perf_event_alloc(struct perf_event_attr event->attr = *attr; event->group_leader = group_leader; event->pmu =NULL ; - event->ctx = ctx; event->oncpu =-1 ; event->parent = parent_event;
@@ -5320,20 +5318,26 @@ SYSCALL_DEFINE5(perf_event_open,if (event_fd <0 )return event_fd; + event =perf_event_alloc (&attr, cpu, group_leader,NULL ,NULL ); +if (IS_ERR (event)) { + err =PTR_ERR (event); + goto err_fd; + } +/* * Get the target context (task or percpu): */ ctx =find_get_context (pid, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_fd; + goto err_alloc; }if (group_fd !=-1 ) { group_leader =perf_fget_light (group_fd, &fput_needed);if (IS_ERR (group_leader)) { err =PTR_ERR (group_leader); - goto err_put_context; + goto err_context; } group_file = group_leader->filp;if (flags &PERF_FLAG_FD_OUTPUT )
@@ -5353,37 +5357,30 @@ SYSCALL_DEFINE5(perf_event_open,* becoming part of another group-sibling): */ if (group_leader->group_leader != group_leader) - goto err_put_context; + goto err_context;/* * Do not allow to attach to a group in a different * task or CPU context: */ if (group_leader->ctx != ctx) - goto err_put_context; + goto err_context;/* * Only a group leader can be exclusive or pinned */ if (attr.exclusive || attr.pinned) - goto err_put_context; - } - - event =perf_event_alloc (&attr, cpu, ctx, group_leader, -NULL ,NULL ,GFP_KERNEL ); -if (IS_ERR (event)) { - err =PTR_ERR (event); - goto err_put_context; + goto err_context; }if (output_event) { err =perf_event_set_output (event, output_event);if (err) - goto err_free_put_context; + goto err_context; } event_file =anon_inode_getfile ("[perf_event]" , &perf_fops, event,O_RDWR );if (IS_ERR (event_file)) { err =PTR_ERR (event_file); - goto err_free_put_context; + goto err_context; } event->filp = event_file;
@@ -5409,11 +5406,11 @@ SYSCALL_DEFINE5(perf_event_open,fd_install (event_fd, event_file);return event_fd; -err_free_put_context: -free_event (event); -err_put_context: +err_context:fput_light (group_file, fput_needed);put_ctx (ctx); +err_alloc: +free_event (event); err_fd:put_unused_fd (event_fd);return err;
@@ -5431,25 +5428,24 @@ perf_event_create_kernel_counter(struct pid_t pid, perf_overflow_handler_t overflow_handler) { -struct perf_event *event;struct perf_event_context *ctx; +struct perf_event *event;int err;/* * Get the target context (task or percpu): */ + event =perf_event_alloc (attr, cpu,NULL ,NULL , overflow_handler); +if (IS_ERR (event)) { + err =PTR_ERR (event); + goto err; + } + ctx =find_get_context (pid, cpu);if (IS_ERR (ctx)) { err =PTR_ERR (ctx); - goto err_exit; - } - - event =perf_event_alloc (attr, cpu, ctx,NULL , -NULL , overflow_handler,GFP_KERNEL ); -if (IS_ERR (event)) { - err =PTR_ERR (event); - goto err_put_context; + goto err_free; } event->filp =NULL ;
@@ -5467,9 +5463,9 @@ perf_event_create_kernel_counter(structreturn event; - err_put_context: -put_ctx (ctx); - err_exit: +err_free: +free_event (event); +err:return ERR_PTR (err); }EXPORT_SYMBOL_GPL (perf_event_create_kernel_counter);
@@ -5497,9 +5493,9 @@ inherit_event(struct perf_event *parent_ parent_event = parent_event->parent; child_event =perf_event_alloc (&parent_event->attr, - parent_event->cpu, child_ctx, + parent_event->cpu, group_leader, parent_event, -NULL ,GFP_KERNEL ); +NULL );if (IS_ERR (child_event))return child_event;get_ctx (child_ctx);
@@ -5524,6 +5520,7 @@ inherit_event(struct perf_event *parent_local64_set (&hwc->period_left, sample_period); } + child_event->ctx = child_ctx; child_event->overflow_handler = parent_event->overflow_handler;/*
RFC PATCH 14/19 perf: Per-pmu-per-cpu contexts
In reply to Peter ZijlstraAllocate per-cpu contexts per pmu.
--- include/linux/perf_event.h | 4 - kernel/perf_event.c | 159 +++++++++++++++++++++++++++------------------ 2 files changed, 101 insertions(+), 62 deletions(-)
include/linux/perf_event.h +3 -1
@@ -570,7 +570,8 @@ struct perf_event;struct pmu {struct list_head entry; -int *pmu_disable_count; +int *__percpu pmu_disable_count; +struct perf_cpu_context *__percpu pmu_cpu_context;/* * Fully disable/enable this PMU, can be used to protect from the PMI
@@ -813,6 +814,7 @@ struct perf_event {* Used as a container for task events and CPU events as well: */ struct perf_event_context { +struct pmu *pmu;/* * Protect the states of the events in the list, * nr_active, and the list:
kernel/perf_event.c +98 -61
398 lines not on this page · raw message
RFC PATCH 16/19 perf: Clean up perf_event_context allocation
In reply to Peter ZijlstraUnify the two perf_event_context allocation sites.
--- kernel/perf_event.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-)
kernel/perf_event.c +26 -15
@@ -1936,9 +1936,7 @@ static struct perf_callchain_entry *perf/* * Initialize the perf_event context in a task_struct: */ -static void -__perf_event_init_context (struct perf_event_context *ctx, -struct task_struct *task) +static void __perf_event_init_context (struct perf_event_context *ctx) {raw_spin_lock_init (&ctx->lock);mutex_init (&ctx->mutex);
@@ -1946,7 +1944,25 @@ __perf_event_init_context(struct perf_evINIT_LIST_HEAD (&ctx->flexible_groups);INIT_LIST_HEAD (&ctx->event_list);atomic_set (&ctx->refcount,1 ); - ctx->task = task; +} + +static struct perf_event_context * +alloc_perf_context (struct pmu *pmu,struct task_struct *task) +{ +struct perf_event_context *ctx; + + ctx =kzalloc (sizeof (struct perf_event_context ),GFP_KERNEL ); +if (!ctx) +return NULL ; + +__perf_event_init_context (ctx); +if (task) { + ctx->task = task; +get_task_struct (task); + } + ctx->pmu = pmu; + +return ctx; } static struct perf_event_context *
@@ -2013,22 +2029,22 @@ find_get_default_context(struct pmu *pmu }if (!ctx) { - ctx =kzalloc (sizeof (struct perf_event_context ),GFP_KERNEL ); + ctx =alloc_perf_context (pmu, task); err = -ENOMEM ;if (!ctx) goto errout; -__perf_event_init_context (ctx, task); - ctx->pmu = pmu; +get_ctx (ctx); +if (cmpxchg (&task->perf_event_ctxp,NULL , ctx)) {/* * We raced with some other task; use * the context they set. */ +put_task_struct (task);kfree (ctx); goto retry; } -get_task_struct (task); }put_task_struct (task);
@@ -5021,7 +5037,7 @@ int perf_pmu_register(struct pmu *pmu)struct perf_cpu_context *cpuctx; cpuctx =per_cpu_ptr (pmu->pmu_cpu_context, cpu); -__perf_event_init_context (&cpuctx->ctx,NULL ); +__perf_event_init_context (&cpuctx->ctx); cpuctx->ctx.pmu = pmu; cpuctx->timer_interval =TICK_NSEC ;hrtimer_init (&cpuctx->timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL );
@@ -5848,15 +5864,11 @@ inherit_task_group(struct perf_event *ev* child. */ - child_ctx =kzalloc (sizeof (struct perf_event_context ), -GFP_KERNEL ); + child_ctx =alloc_perf_context (event->pmu, child);if (!child_ctx)return -ENOMEM ; -__perf_event_init_context (child_ctx, child); - child_ctx->pmu = event->pmu; child->perf_event_ctxp = child_ctx; -get_task_struct (child); } ret = inherit_group(event, parent, parent_ctx,
@@ -5868,7 +5880,6 @@ inherit_task_group(struct perf_event *evreturn ret; } -/* * Initialize the perf_event context in task_struct */
RFC PATCH 7/19 perf: Default PMU ops
In reply to Peter ZijlstraProvide default implementations for the pmu txn methods, this allows us to remove some conditional code.
--- include/linux/perf_event.h | 10 +++---- kernel/perf_event.c | 64 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 17 deletions(-)
kernel/perf_event.c +52 -12
@@ -656,21 +656,14 @@ group_sched_in(struct perf_event *group_ {struct perf_event *event, *partial_group =NULL ;struct pmu *pmu = group_event->pmu; -bool txn = false;if (group_event->state ==PERF_EVENT_STATE_OFF )return 0 ; -/* Check if group transaction availabe */ -if (pmu->start_txn) - txn = true; - -if (txn) - pmu->start_txn (pmu); + pmu->start_txn (pmu);if (event_sched_in (group_event, cpuctx, ctx)) { -if (txn) - pmu->cancel_txn (pmu); + pmu->cancel_txn (pmu);return -EAGAIN ; }
@@ -684,7 +677,7 @@ group_sched_in(struct perf_event *group_ } } -if (!txn || !pmu->commit_txn (pmu)) +if (!pmu->commit_txn (pmu))return 0 ; group_error:
@@ -699,8 +692,7 @@ group_error: }event_sched_out (group_event, cpuctx, ctx); -if (txn) - pmu->cancel_txn (pmu); + pmu->cancel_txn (pmu);return -EAGAIN ; }
@@ -4755,6 +4747,31 @@ static LIST_HEAD(pmus);static DEFINE_MUTEX (pmus_lock);static struct srcu_struct pmus_srcu; +static void perf_pmu_nop_void (struct pmu *pmu) +{ +} + +static int perf_pmu_nop_int (struct pmu *pmu) +{ +return 0 ; +} + +static void perf_pmu_start_txn (struct pmu *pmu) +{ +perf_pmu_disable (pmu); +} + +static int perf_pmu_commit_txn (struct pmu *pmu) +{ +perf_pmu_enable (pmu); +return 0 ; +} + +static void perf_pmu_cancel_txn (struct pmu *pmu) +{ +perf_pmu_enable (pmu); +} +int perf_pmu_register (struct pmu *pmu) {int ret;
@@ -4764,6 +4781,29 @@ int perf_pmu_register(struct pmu *pmu) pmu->pmu_disable_count =alloc_percpu (int);if (!pmu->pmu_disable_count) goto unlock; + +if (!pmu->start_txn) { +if (pmu->pmu_enable) { +/* +* If we have pmu_enable/pmu_disable calls, install +* transaction stubs that use that to try and batch +* hardware accesses. +*/ + pmu->start_txn = perf_pmu_start_txn; + pmu->commit_txn = perf_pmu_commit_txn; + pmu->cancel_txn = perf_pmu_cancel_txn; + }else { + pmu->start_txn = perf_pmu_nop_void; + pmu->commit_txn = perf_pmu_nop_int; + pmu->cancel_txn = perf_pmu_nop_void; + } + } + +if (!pmu->pmu_enable) { + pmu->pmu_enable = perf_pmu_nop_void; + pmu->pmu_disable = perf_pmu_nop_void; + } +list_add_rcu (&pmu->entry, &pmus); ret =0 ; unlock:
include/linux/perf_event.h +5 -5
@@ -565,8 +565,8 @@ struct pmu {int *pmu_disable_count; -void (*pmu_enable) (struct pmu *pmu); -void (*pmu_disable) (struct pmu *pmu); +void (*pmu_enable) (struct pmu *pmu);/* optional */ +void (*pmu_disable) (struct pmu *pmu);/* optional */ /* * Should return -ENOENT when the @event doesn't match this PMU.
@@ -590,19 +590,19 @@ struct pmu {* Start the transaction, after this ->enable() doesn't need to * do schedulability tests. */ -void (*start_txn) (struct pmu *pmu); +void (*start_txn) (struct pmu *pmu);/* optional */ /* * If ->start_txn() disabled the ->enable() schedulability test * then ->commit_txn() is required to perform one. On success * the transaction is closed. On error the transaction is kept * open until ->cancel_txn() is called. */ -int (*commit_txn) (struct pmu *pmu); +int (*commit_txn) (struct pmu *pmu);/* optional */ /* * Will cancel the transaction, assumes ->disable() is called * for each successfull ->enable() during the transaction. */ -void (*cancel_txn) (struct pmu *pmu); +void (*cancel_txn) (struct pmu *pmu);/* optional */ };/**
RFC PATCH 6/19 perf: Per PMU disable
In reply to Peter ZijlstraChanges perf_disable() into perf_pmu_disable().
--- arch/alpha/kernel/perf_event.c | 30 +++++++++++++----------- arch/arm/kernel/perf_event.c | 28 +++++++++++----------- arch/powerpc/kernel/perf_event.c | 24 ++++++++++--------- arch/powerpc/kernel/perf_event_fsl_emb.c | 18 ++++++++------ arch/sh/kernel/perf_event.c | 38 ++++++++++++++++--------------- arch/sparc/kernel/perf_event.c | 20 ++++++++-------- arch/x86/kernel/cpu/perf_event.c | 16 +++++++------ include/linux/perf_event.h | 13 +++++----- kernel/perf_event.c | 31 +++++++++++++++---------- 9 files changed, 119 insertions(+), 99 deletions(-)
arch/arm/kernel/perf_event.c +14 -14
@@ -277,7 +277,7 @@ armpmu_enable(struct perf_event *event)int idx;int err =0 ; -perf_disable (); +perf_pmu_disable (event->pmu);/* If we don't have a space for the counter then finish early. */ idx = armpmu->get_event_idx (cpuc, hwc);
@@ -305,7 +305,7 @@ armpmu_enable(struct perf_event *event)perf_event_update_userpage (event); out: -perf_enable (); +perf_pmu_enable (event->pmu);return err; }
@@ -534,16 +534,7 @@ static int armpmu_event_init(struct perfreturn err; } -static struct pmu pmu = { - .event_init = armpmu_event_init, - .enable = armpmu_enable, - .disable = armpmu_disable, - .unthrottle = armpmu_unthrottle, - .read = armpmu_read, -}; - -void -hw_perf_enable (void ) +static void armpmu_pmu_enable (struct pmu *pmu) {/* Enable all of the perf events on hardware. */ int idx;
@@ -564,13 +555,22 @@ hw_perf_enable(void) armpmu->start (); } -void -hw_perf_disable (void ) +static void armpmu_pmu_disable (struct pmu *pmu) {if (armpmu) armpmu->stop (); } +static struct pmu pmu = { + .pmu_enable = armpmu_pmu_enable, + .pmu_disable= armpmu_pmu_disable, + .event_init = armpmu_event_init, + .enable = armpmu_enable, + .disable = armpmu_disable, + .unthrottle = armpmu_unthrottle, + .read = armpmu_read, +}; +/* * ARMv6 Performance counter handling code. *
arch/powerpc/kernel/perf_event.c +13 -11
@@ -517,7 +517,7 @@ static void write_mmcr0(struct cpu_hw_ev* Disable all events to prevent PMU interrupts and to allow * events to be added or removed. */ -void hw_perf_disable (void ) +static void power_pmu_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuhw;unsigned long flags;
@@ -565,7 +565,7 @@ void hw_perf_disable(void)* If we were previously disabled and events were added, then * put the new config on the PMU. */ -void hw_perf_enable (void ) +static void power_pmu_pmu_enable (struct pmu *pmu) {struct perf_event *event;struct cpu_hw_events *cpuhw;
@@ -735,7 +735,7 @@ static int power_pmu_enable(struct perf_int ret = -EAGAIN ;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu);/* * Add the event to the list (if there is room)
@@ -769,7 +769,7 @@ static int power_pmu_enable(struct perf_ ret =0 ; out: -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags);return ret; }
@@ -784,7 +784,7 @@ static void power_pmu_disable(struct perunsigned long flags;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu);power_pmu_read (event);
@@ -821,7 +821,7 @@ static void power_pmu_disable(struct per cpuhw->mmcr[0 ] &= ~(MMCR0_PMXE |MMCR0_FCECE ); } -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags); }
@@ -837,7 +837,7 @@ static void power_pmu_unthrottle(structif (!event->hw.idx || !event->hw.sample_period)return ;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu);power_pmu_read (event); left = event->hw.sample_period; event->hw.last_period = left;
@@ -848,7 +848,7 @@ static void power_pmu_unthrottle(structlocal64_set (&event->hw.prev_count, val);local64_set (&event->hw.period_left, left);perf_event_update_userpage (event); -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags); }
@@ -861,7 +861,7 @@ void power_pmu_start_txn(struct pmu *pmu {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); -perf_disable (); +perf_pmu_disable (pmu); cpuhw->group_flag |=PERF_EVENT_TXN ; cpuhw->n_txn_start = cpuhw->n_events; }
@@ -876,7 +876,7 @@ void power_pmu_cancel_txn(struct pmu *pmstruct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ; -perf_enable (); +perf_pmu_enable (pmu); }/*
@@ -903,7 +903,7 @@ int power_pmu_commit_txn(struct pmu *pmu cpuhw->event[i]->hw.config = cpuhw->events[i]; cpuhw->group_flag &= ~PERF_EVENT_TXN ; -perf_enable (); +perf_pmu_enable (pmu);return 0 ; }
@@ -1131,6 +1131,8 @@ static int power_pmu_event_init(struct p }struct pmu power_pmu = { + .pmu_enable = power_pmu_pmu_enable, + .pmu_disable = power_pmu_pmu_disable, .event_init = power_pmu_event_init, .enable = power_pmu_enable, .disable = power_pmu_disable,
arch/powerpc/kernel/perf_event_fsl_emb.c +10 -8
@@ -177,7 +177,7 @@ static void fsl_emb_pmu_read(struct perf* Disable all events to prevent PMU interrupts and to allow * events to be added or removed. */ -void hw_perf_disable (void ) +static void fsl_emb_pmu_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuhw;unsigned long flags;
@@ -216,7 +216,7 @@ void hw_perf_disable(void)* If we were previously disabled and events were added, then * put the new config on the PMU. */ -void hw_perf_enable (void ) +static void fsl_emb_pmu_pmu_enable (struct pmu *pmu) {struct cpu_hw_events *cpuhw;unsigned long flags;
@@ -271,7 +271,7 @@ static int fsl_emb_pmu_enable(struct peru64 val;int i; -perf_disable (); +perf_pmu_disable (event->pmu); cpuhw = &get_cpu_var (cpu_hw_events);if (event->hw.config &FSL_EMB_EVENT_RESTRICTED )
@@ -311,7 +311,7 @@ static int fsl_emb_pmu_enable(struct per ret =0 ; out:put_cpu_var (cpu_hw_events); -perf_enable (); +perf_pmu_enable (event->pmu);return ret; }
@@ -321,7 +321,7 @@ static void fsl_emb_pmu_disable(struct pstruct cpu_hw_events *cpuhw;int i = event->hw.idx; -perf_disable (); +perf_pmu_disable (event->pmu);if (i <0 ) goto out;
@@ -349,7 +349,7 @@ static void fsl_emb_pmu_disable(struct p cpuhw->n_events--; out: -perf_enable (); +perf_pmu_enable (event->pmu);put_cpu_var (cpu_hw_events); }
@@ -367,7 +367,7 @@ static void fsl_emb_pmu_unthrottle(strucif (event->hw.idx <0 || !event->hw.sample_period)return ;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu);fsl_emb_pmu_read (event); left = event->hw.sample_period; event->hw.last_period = left;
@@ -378,7 +378,7 @@ static void fsl_emb_pmu_unthrottle(struclocal64_set (&event->hw.prev_count, val);local64_set (&event->hw.period_left, left);perf_event_update_userpage (event); -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags); }
@@ -524,6 +524,8 @@ static int fsl_emb_pmu_event_init(struct }static struct pmu fsl_emb_pmu = { + .pmu_enable = fsl_emb_pmu_pmu_enable, + .pmu_disable = fsl_emb_pmu_pmu_disable, .event_init = fsl_emb_pmu_event_init, .enable = fsl_emb_pmu_enable, .disable = fsl_emb_pmu_disable,
arch/sh/kernel/perf_event.c +20 -18
@@ -232,7 +232,7 @@ static int sh_pmu_enable(struct perf_eveint idx = hwc->idx;int ret = -EAGAIN ; -perf_disable (); +perf_pmu_disable (event->pmu);if (test_and_set_bit (idx, cpuc->used_mask)) { idx =find_first_zero_bit (cpuc->used_mask, sh_pmu->num_events);
@@ -253,7 +253,7 @@ static int sh_pmu_enable(struct perf_eveperf_event_update_userpage (event); ret =0 ; out: -perf_enable (); +perf_pmu_enable (event->pmu);return ret; }
@@ -285,7 +285,25 @@ static int sh_pmu_event_init(struct perfreturn err; } +static void sh_pmu_pmu_enable (struct pmu *pmu) +{ +if (!sh_pmu_initialized ()) +return ; + + sh_pmu->enable_all (); +} + +static void sh_pmu_pmu_disable (struct pmu *pmu) +{ +if (!sh_pmu_initialized ()) +return ; + + sh_pmu->disable_all (); +} +static struct pmu pmu = { + .pmu_enable = sh_pmu_pmu_enable, + .pmu_disable = sh_pmu_pmu_disable, .event_init = sh_pmu_event_init, .enable = sh_pmu_enable, .disable = sh_pmu_disable,
@@ -316,22 +334,6 @@ sh_pmu_notifier(struct notifier_block *sreturn NOTIFY_OK ; } -void hw_perf_enable (void ) -{ -if (!sh_pmu_initialized ()) -return ; - - sh_pmu->enable_all (); -} - -void hw_perf_disable (void ) -{ -if (!sh_pmu_initialized ()) -return ; - - sh_pmu->disable_all (); -} -int __cpuinit register_sh_pmu (struct sh_pmu *pmu) {if (sh_pmu)
arch/sparc/kernel/perf_event.c +11 -9
@@ -664,7 +664,7 @@ static u64 maybe_change_configuration(streturn pcr; } -void hw_perf_enable (void ) +static void sparc_pmu_pmu_enable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);u64 pcr;
@@ -691,7 +691,7 @@ void hw_perf_enable(void) pcr_ops->write (cpuc->pcr); } -void hw_perf_disable (void ) +static void sparc_pmu_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);u64 val;
@@ -718,7 +718,7 @@ static void sparc_pmu_disable(struct perint i;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu);for (i =0 ; i < cpuc->n_events; i++) {if (event == cpuc->event[i]) {
@@ -748,7 +748,7 @@ static void sparc_pmu_disable(struct per } } -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags); }
@@ -991,7 +991,7 @@ static int sparc_pmu_enable(struct perf_unsigned long flags;local_irq_save (flags); -perf_disable (); +perf_pmu_disable (event->pmu); n0 = cpuc->n_events;if (n0 >= perf_max_events)
@@ -1020,7 +1020,7 @@ static int sparc_pmu_enable(struct perf_ ret =0 ; out: -perf_enable (); +perf_pmu_enable (event->pmu);local_irq_restore (flags);return ret; }
@@ -1113,7 +1113,7 @@ static void sparc_pmu_start_txn(struct p {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); -perf_disable (); +perf_pmu_disable (pmu); cpuhw->group_flag |=PERF_EVENT_TXN ; }
@@ -1127,7 +1127,7 @@ static void sparc_pmu_cancel_txn(structstruct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events); cpuhw->group_flag &= ~PERF_EVENT_TXN ; -perf_enable (); +perf_pmu_enable (pmu); }/*
@@ -1151,11 +1151,13 @@ static int sparc_pmu_commit_txn(struct preturn -EAGAIN ; cpuc->group_flag &= ~PERF_EVENT_TXN ; -perf_enable (); +perf_pmu_enable (pmu);return 0 ; }static struct pmu pmu = { + .pmu_enable = sparc_pmu_pmu_enable, + .pmu_disable = sparc_pmu_pmu_disable, .event_init = sparc_pmu_event_init, .enable = sparc_pmu_enable, .disable = sparc_pmu_disable,
arch/x86/kernel/cpu/perf_event.c +9 -7
@@ -582,7 +582,7 @@ static void x86_pmu_disable_all(void) } } -void hw_perf_disable (void ) +static void x86_pmu_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -802,7 +802,7 @@ static inline int match_prev_assignment(static int x86_pmu_start (struct perf_event *event);static void x86_pmu_stop (struct perf_event *event); -void hw_perf_enable (void ) +static void x86_pmu_pmu_enable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct perf_event *event;
@@ -968,7 +968,7 @@ static int x86_pmu_enable(struct perf_ev hwc = &event->hw; -perf_disable (); +perf_pmu_disable (event->pmu); n0 = cpuc->n_events; ret = n =collect_events (cpuc, event, false);if (ret <0 )
@@ -998,7 +998,7 @@ static int x86_pmu_enable(struct perf_ev ret =0 ; out: -perf_enable (); +perf_pmu_enable (event->pmu);return ret; }
@@ -1435,7 +1435,7 @@ static void x86_pmu_start_txn(struct pmu {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); -perf_disable (); +perf_pmu_disable (pmu); cpuc->group_flag |=PERF_EVENT_TXN ; cpuc->n_txn =0 ; }
@@ -1455,7 +1455,7 @@ static void x86_pmu_cancel_txn(struct pm*/ cpuc->n_added -= cpuc->n_txn; cpuc->n_events -= cpuc->n_txn; -perf_enable (); +perf_pmu_enable (pmu); }/*
@@ -1485,7 +1485,7 @@ static int x86_pmu_commit_txn(struct pmumemcpy (cpuc->assign, assign, n*sizeof (int )); cpuc->group_flag &= ~PERF_EVENT_TXN ; -perf_enable (); +perf_pmu_enable (pmu);return 0 ; }
@@ -1604,6 +1604,8 @@ int x86_pmu_event_init(struct perf_event }static struct pmu pmu = { + .pmu_enable = x86_pmu_pmu_enable, + .pmu_disable = x86_pmu_pmu_disable, .event_init = x86_pmu_event_init, .enable = x86_pmu_enable, .disable = x86_pmu_disable,
include/linux/perf_event.h +7 -6
@@ -563,6 +563,11 @@ struct perf_event;struct pmu {struct list_head entry; +int *pmu_disable_count; + +void (*pmu_enable) (struct pmu *pmu); +void (*pmu_disable) (struct pmu *pmu); +/* * Should return -ENOENT when the @event doesn't match this PMU. */
@@ -868,10 +873,8 @@ extern void perf_event_free_task(structextern void set_perf_event_pending (void );extern void perf_event_do_pending (void );extern void perf_event_print_debug (void ); -extern void __perf_disable (void ); -extern bool __perf_enable (void ); -extern void perf_disable (void ); -extern void perf_enable (void ); +extern void perf_pmu_disable (struct pmu *pmu); +extern void perf_pmu_enable (struct pmu *pmu);extern int perf_event_task_disable (void );extern int perf_event_task_enable (void );extern void perf_event_update_userpage (struct perf_event *event);
@@ -1056,8 +1059,6 @@ static inline void perf_event_exit_task(static inline void perf_event_free_task (struct task_struct *task) { }static inline void perf_event_do_pending (void ) { }static inline void perf_event_print_debug (void ) { } -static inline void perf_disable (void ) { } -static inline void perf_enable (void ) { }static inline int perf_event_task_disable (void ) {return -EINVAL ; }static inline int perf_event_task_enable (void ) {return -EINVAL ; }
kernel/perf_event.c +19 -12
@@ -71,23 +71,20 @@ static atomic64_t perf_event_id;*/ static DEFINE_SPINLOCK (perf_resource_lock); -void __weak hw_perf_disable (void ) {barrier (); } -void __weak hw_perf_enable (void ) {barrier (); } -void __weak perf_event_print_debug (void ) { } -static DEFINE_PER_CPU(int, perf_disable_count); - -void perf_disable (void ) +void perf_pmu_disable (struct pmu *pmu) { -if (!__get_cpu_var (perf_disable_count)++) -hw_perf_disable (); +int *count =this_cpu_ptr (pmu->pmu_disable_count); +if (!(*count)++) + pmu->pmu_disable (pmu); } -void perf_enable (void ) +void perf_pmu_enable (struct pmu *pmu) { -if (!--__get_cpu_var (perf_disable_count)) -hw_perf_enable (); +int *count =this_cpu_ptr (pmu->pmu_disable_count); +if (!--(*count)) + pmu->pmu_enable (pmu); }static void get_ctx (struct perf_event_context *ctx)
@@ -4970,11 +4967,19 @@ static struct srcu_struct pmus_srcu;int perf_pmu_register (struct pmu *pmu) { +int ret; +mutex_lock (&pmus_lock); + ret = -ENOMEM ; + pmu->pmu_disable_count =alloc_percpu (int); +if (!pmu->pmu_disable_count) + goto unlock;list_add_rcu (&pmu->entry, &pmus); + ret =0 ; +unlock:mutex_unlock (&pmus_lock); -return 0 ; +return ret; }void perf_pmu_unregister (struct pmu *pmu)
@@ -4984,6 +4989,8 @@ void perf_pmu_unregister(struct pmu *pmumutex_unlock (&pmus_lock);synchronize_srcu (&pmus_srcu); + +free_percpu (pmu->pmu_disable_count); }struct pmu *perf_init_event (struct perf_event *event)
arch/alpha/kernel/perf_event.c +16 -14
@@ -435,7 +435,7 @@ static int alpha_pmu_enable(struct perf_* nevertheless we disable the PMCs first to enable a potential * final PMI to occur before we disable interrupts. */ -perf_disable (); +perf_pmu_disable (event->pmu);local_irq_save (flags);/* Default to error to be returned */
@@ -456,7 +456,7 @@ static int alpha_pmu_enable(struct perf_ }local_irq_restore (flags); -perf_enable (); +perf_pmu_enable (event->pmu);return ret; }
@@ -474,7 +474,7 @@ static void alpha_pmu_disable(struct perunsigned long flags;int j; -perf_disable (); +perf_pmu_disable (event->pmu);local_irq_save (flags);for (j =0 ; j < cpuc->n_events; j++) {
@@ -502,7 +502,7 @@ static void alpha_pmu_disable(struct per }local_irq_restore (flags); -perf_enable (); +perf_pmu_enable (event->pmu); }
@@ -668,18 +668,10 @@ static int alpha_pmu_event_init(struct preturn err; } -static struct pmu pmu = { - .event_init = alpha_pmu_event_init, - .enable = alpha_pmu_enable, - .disable = alpha_pmu_disable, - .read = alpha_pmu_read, - .unthrottle = alpha_pmu_unthrottle, -}; -/* * Main entry point - enable HW performance counters. */ -void hw_perf_enable (void ) +static void alpha_pmu_pmu_enable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -705,7 +697,7 @@ void hw_perf_enable(void)* Main entry point - disable HW performance counters. */ -void hw_perf_disable (void ) +static void alpha_pmu_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -718,6 +710,16 @@ void hw_perf_disable(void)wrperfmon (PERFMON_CMD_DISABLE , cpuc->idx_mask); } +static struct pmu pmu = { + .pmu_enable = alpha_pmu_pmu_enable, + .pmu_disable = alpha_pmu_pmu_disable, + .event_init = alpha_pmu_event_init, + .enable = alpha_pmu_enable, + .disable = alpha_pmu_disable, + .read = alpha_pmu_read, + .unthrottle = alpha_pmu_unthrottle, +}; +/* * Main entry point - don't know when this is called but it
RFC PATCH 9/19 perf: Rework the PMU methods
In reply to Peter ZijlstraReplace pmu::{enable,disable,start,stop,unthrottle} with pmu::{add,del,start,stop}, all of which take a flags argument.
The new interface extends the capability to stop a counter while keeping it scheduled on the PMU. We replace the throttled state with the generic stopped state.
This also allows us to efficiently stop/start counters over certain code paths (like IRQ handlers).
It also allows scheduling a counter without it starting, allowing for a generic frozen state (useful for rotating stopped counters).
The stopped state is implemented in two different ways, depending on how the architecture implemented the throttled state:
1) We disable the counter:
a) the pmu has per-counter enable bits, we flip that
b) we program a NOP event, preserving the counter state
2) We store the counter state and ignore all read/overflow events
--- arch/alpha/kernel/perf_event.c | 71 +++++++++++---- arch/arm/kernel/perf_event.c | 96 +++++++++++++------- arch/powerpc/kernel/perf_event.c | 105 ++++++++++++++-------- arch/powerpc/kernel/perf_event_fsl_emb.c | 107 ++++++++++++++-------- arch/sh/kernel/perf_event.c | 75 ++++++++++------ arch/sparc/kernel/perf_event.c | 109 ++++++++++++++--------- arch/x86/kernel/cpu/perf_event.c | 106 +++++++++++++--------- arch/x86/kernel/cpu/perf_event_intel.c | 2 arch/x86/kernel/cpu/perf_event_intel_ds.c | 2 include/linux/ftrace_event.h | 4 include/linux/perf_event.h | 54 ++++++++--- kernel/hw_breakpoint.c | 29 +++++- kernel/perf_event.c | 140 +++++++++++++++--------------- kernel/trace/trace_event_perf.c | 7 + 14 files changed, 576 insertions(+), 331 deletions(-)
arch/x86/kernel/cpu/perf_event.c +61 -45
@@ -582,7 +582,7 @@ static void x86_pmu_disable_all(void) } } -static void x86_pmu_pmu_disable (struct pmu *pmu) +static void x86_pmu_disable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -799,10 +799,10 @@ static inline int match_prev_assignment( hwc->last_tag == cpuc->tags[i]; } -static int x86_pmu_start (struct perf_event *event); -static void x86_pmu_stop (struct perf_event *event); +static void x86_pmu_start (struct perf_event *event,int flags); +static void x86_pmu_stop (struct perf_event *event,int flags); -static void x86_pmu_pmu_enable (struct pmu *pmu) +static void x86_pmu_enable (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct perf_event *event;
@@ -838,7 +838,14 @@ static void x86_pmu_pmu_enable(struct pm match_prev_assignment(hwc, cpuc, i))continue ; -x86_pmu_stop (event); +/* +* Ensure we don't accidentally enable a stopped +* counter simply because we rescheduled. +*/ +if (hwc->state &PERF_HES_STOPPED ) + hwc->state |=PERF_HES_ARCH ; + +x86_pmu_stop (event,PERF_EF_UPDATE ); }for (i =0 ; i < cpuc->n_events; i++) {
@@ -850,7 +857,10 @@ static void x86_pmu_pmu_enable(struct pmelse if (i < n_running)continue ; -x86_pmu_start (event); +if (hwc->state &PERF_HES_ARCH ) +continue ; + +x86_pmu_start (event,PERF_EF_RELOAD ); } cpuc->n_added =0 ;perf_events_lapic_init ();
@@ -951,15 +961,12 @@ static void x86_pmu_enable_event(struct }/* -* activate a single event +* Add a single event to the PMU. * * The event is added to the group of enabled events * but only if it can be scehduled with existing events. -* -* Called with PMU disabled. If successful and return value 1, -* then guaranteed to call perf_enable() and hw_perf_enable() */ -static int x86_pmu_enable (struct perf_event *event) +static int x86_pmu_add (struct perf_event *event,int flags) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct hw_perf_event *hwc;
@@ -974,10 +981,14 @@ static int x86_pmu_enable(struct perf_evif (ret <0 ) goto out; + hwc->state =PERF_HES_UPTODATE |PERF_HES_STOPPED ; +if (!(flags &PERF_EF_START )) + hwc->state |=PERF_HES_ARCH ; +/* * If group events scheduling transaction was started, * skip the schedulability test here, it will be peformed -* at commit time(->commit_txn) as a whole +* at commit time (->commit_txn) as a whole */ if (cpuc->group_flag &PERF_EVENT_TXN ) goto done_collect;
@@ -1002,27 +1013,28 @@ static int x86_pmu_enable(struct perf_evreturn ret; } -static int x86_pmu_start (struct perf_event *event) +static void x86_pmu_start (struct perf_event *event,int flags) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);int idx = event->hw.idx; -if (idx ==-1 ) -return -EAGAIN ; +if (WARN_ON_ONCE (!(event->hw.state &PERF_HES_STOPPED ))) +return ; + +if (WARN_ON_ONCE (idx ==-1 )) +return ; + +if (flags &PERF_EF_RELOAD ) { +WARN_ON_ONCE (!(event->hw.state &PERF_HES_UPTODATE )); +x86_perf_event_set_period (event); + } + + event->hw.state =0 ; -x86_perf_event_set_period (event); cpuc->events[idx] = event;__set_bit (idx, cpuc->active_mask); x86_pmu.enable (event);perf_event_update_userpage (event); - -return 0 ; -} - -static void x86_pmu_unthrottle (struct perf_event *event) -{ -int ret =x86_pmu_start (event); -WARN_ON_ONCE (ret); }void perf_event_print_debug (void )
@@ -1079,27 +1091,29 @@ void perf_event_print_debug(void)local_irq_restore (flags); } -static void x86_pmu_stop (struct perf_event *event) +static void x86_pmu_stop (struct perf_event *event,int flags) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct hw_perf_event *hwc = &event->hw; -int idx = hwc->idx; - -if (!__test_and_clear_bit (idx, cpuc->active_mask)) -return ; - - x86_pmu.disable (event); -/* -* Drain the remaining delta count out of a event -* that we are disabling: -*/ -x86_perf_event_update (event); +if (__test_and_clear_bit (hwc->idx, cpuc->active_mask)) { + x86_pmu.disable (event); + cpuc->events[hwc->idx] =NULL ; +WARN_ON_ONCE (hwc->state &PERF_HES_STOPPED ); + hwc->state |=PERF_HES_STOPPED ; + } - cpuc->events[idx] =NULL ; +if ((flags &PERF_EF_UPDATE ) && !(hwc->state &PERF_HES_UPTODATE )) { +/* +* Drain the remaining delta count out of a event +* that we are disabling: +*/ +x86_perf_event_update (event); + hwc->state |=PERF_HES_UPTODATE ; + } } -static void x86_pmu_disable (struct perf_event *event) +static void x86_pmu_del (struct perf_event *event,int flags) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);int i;
@@ -1112,7 +1126,7 @@ static void x86_pmu_disable(struct perf_if (cpuc->group_flag &PERF_EVENT_TXN )return ; -x86_pmu_stop (event); +x86_pmu_stop (event,PERF_EF_UPDATE );for (i =0 ; i < cpuc->n_events; i++) {if (event == cpuc->event_list[i]) {
@@ -1164,7 +1178,7 @@ static int x86_pmu_handle_irq(struct pt_continue ;if (perf_event_overflow (event,1 , &data, regs)) -x86_pmu_stop (event); +x86_pmu_stop (event,0 ); }if (handled)
@@ -1604,15 +1618,17 @@ int x86_pmu_event_init(struct perf_event }static struct pmu pmu = { - .pmu_enable = x86_pmu_pmu_enable, - .pmu_disable = x86_pmu_pmu_disable, + .pmu_enable = x86_pmu_enable, + .pmu_disable = x86_pmu_disable, + .event_init = x86_pmu_event_init, - .enable = x86_pmu_enable, - .disable = x86_pmu_disable, + + .add = x86_pmu_add, + .del = x86_pmu_del, .start = x86_pmu_start, .stop = x86_pmu_stop, .read = x86_pmu_read, - .unthrottle = x86_pmu_unthrottle, + .start_txn = x86_pmu_start_txn, .cancel_txn = x86_pmu_cancel_txn, .commit_txn = x86_pmu_commit_txn,
kernel/perf_event.c +72 -68
301 lines not on this page · raw message
kernel/trace/trace_event_perf.c +5 -2
@@ -101,7 +101,7 @@ int perf_trace_init(struct perf_event *preturn ret; } -int perf_trace_enable (struct perf_event *p_event) +int perf_trace_add (struct perf_event *p_event,int flags) {struct ftrace_event_call *tp_event = p_event->tp_event;struct hlist_head __percpu *pcpu_list;
@@ -111,13 +111,16 @@ int perf_trace_enable(struct perf_eventif (WARN_ON_ONCE (!pcpu_list))return -EINVAL ; +if (!(flags &PERF_EF_START )) + p_event->hw.state =PERF_HES_STOPPED ; + list =this_cpu_ptr (pcpu_list);hlist_add_head_rcu (&p_event->hlist_entry, list);return 0 ; } -void perf_trace_disable (struct perf_event *p_event) +void perf_trace_del (struct perf_event *p_event,int flags) {hlist_del_rcu (&p_event->hlist_entry); }
arch/sh/kernel/perf_event.c +51 -24
138 lines not on this page · raw message
arch/arm/kernel/perf_event.c +61 -35
153 lines not on this page · raw message
arch/powerpc/kernel/perf_event.c +68 -37
189 lines not on this page · raw message
arch/powerpc/kernel/perf_event_fsl_emb.c +66 -41
184 lines not on this page · raw message
arch/sparc/kernel/perf_event.c +67 -42
184 lines not on this page · raw message
include/linux/ftrace_event.h +2 -2
@@ -252,8 +252,8 @@ DECLARE_PER_CPU(struct pt_regs, perf_traextern int perf_trace_init (struct perf_event *event);extern void perf_trace_destroy (struct perf_event *event); -extern int perf_trace_enable (struct perf_event *event); -extern void perf_trace_disable (struct perf_event *event); +extern int perf_trace_add (struct perf_event *event,int flags); +extern void perf_trace_del (struct perf_event *event,int flags);extern int ftrace_profile_set_filter (struct perf_event *event,int event_id,char *filter_str);extern void ftrace_profile_free_filter (struct perf_event *event);
include/linux/perf_event.h +41 -13
102 lines not on this page · raw message
arch/x86/kernel/cpu/perf_event_intel.c +1 -1
@@ -763,7 +763,7 @@ static int intel_pmu_handle_irq(struct p data.period = event->hw.last_period;if (perf_event_overflow (event,1 , &data, regs)) -x86_pmu_stop (event); +x86_pmu_stop (event,0 ); }/*
arch/x86/kernel/cpu/perf_event_intel_ds.c +1 -1
@@ -491,7 +491,7 @@ static void __intel_pmu_pebs_event(struc regs.flags &= ~PERF_EFLAGS_EXACT ;if (perf_event_overflow (event,1 , &data, ®s)) -x86_pmu_stop (event); +x86_pmu_stop (event,0 ); }static void intel_pmu_drain_pebs_core (struct pt_regs *iregs)
kernel/hw_breakpoint.c +27 -2
@@ -586,10 +586,35 @@ static int hw_breakpoint_event_init(strureturn 0 ; } +static int hw_breakpoint_add (struct perf_event *bp,int flags) +{ +if (!(flags &PERF_EF_START )) + bp->hw.state =PERF_HES_STOPPED ; + +return arch_install_hw_breakpoint (bp); +} + +static void hw_breakpoint_del (struct perf_event *bp,int flags) +{ +arch_uninstall_hw_breakpoint (bp); +} + +static void hw_breakpoint_start (struct perf_event *bp,int flags) +{ + bp->hw.state =0 ; +} + +static void hw_breakpoint_stop (struct perf_event *bp,int flags) +{ + bp->hw.state =PERF_HES_STOPPED ; +} +static struct pmu perf_breakpoint = { .event_init = hw_breakpoint_event_init, - .enable = arch_install_hw_breakpoint, - .disable = arch_uninstall_hw_breakpoint, + .add = hw_breakpoint_add, + .del = hw_breakpoint_del, + .start = hw_breakpoint_start, + .stop = hw_breakpoint_stop, .read = hw_breakpoint_pmu_read, };
arch/alpha/kernel/perf_event.c +53 -18
134 lines not on this page · raw message
RFC PATCH 3/19 perf: register pmu implementations
In reply to Peter ZijlstraSimple registration interface for struct pmu, this provides the infrastructure for removing all the weak functions.
--- arch/alpha/kernel/perf_event.c | 37 + arch/arm/kernel/perf_event.c | 38 +- arch/powerpc/kernel/perf_event.c | 46 +- arch/powerpc/kernel/perf_event_fsl_emb.c | 37 + arch/sh/kernel/perf_event.c | 35 + arch/sparc/kernel/perf_event.c | 29 - arch/x86/kernel/cpu/perf_event.c | 45 +- include/linux/perf_event.h | 10 kernel/hw_breakpoint.c | 35 + kernel/perf_event.c | 588 +++++++++++++++---------------- 10 files changed, 488 insertions(+), 412 deletions(-)
arch/arm/kernel/perf_event.c +26 -12
@@ -306,12 +306,7 @@ out:return err; } -static struct pmu pmu = { - .enable = armpmu_enable, - .disable = armpmu_disable, - .unthrottle = armpmu_unthrottle, - .read = armpmu_read, -}; +static struct pmu pmu; static int validate_event(struct cpu_hw_events *cpuc,
@@ -491,20 +486,29 @@ __hw_perf_event_init(struct perf_event *return err; } -struct pmu * -hw_perf_event_init (struct perf_event *event) +static int armpmu_event_init (struct perf_event *event) {int err =0 ; +switch (event->attr.type) { +case PERF_TYPE_RAW : +case PERF_TYPE_HARDWARE : +case PERF_TYPE_HW_CACHE : +break ; + +default : +return -ENOENT ; + } +if (!armpmu) -return ERR_PTR (-ENODEV ); +return -ENODEV ; event->destroy = hw_perf_event_destroy;if (!atomic_inc_not_zero (&active_events)) {if (atomic_read (&active_events) > perf_max_events) {atomic_dec (&active_events); -return ERR_PTR (-ENOSPC ); +return -ENOSPC ; }mutex_lock (&pmu_reserve_mutex);
@@ -518,15 +522,23 @@ hw_perf_event_init(struct perf_event *ev }if (err) -return ERR_PTR (err); +return err; err =__hw_perf_event_init (event);if (err)hw_perf_event_destroy (event); -return err ?ERR_PTR (err) : &pmu; +return err; } +static struct pmu pmu = { + .event_init = armpmu_event_init, + .enable = armpmu_enable, + .disable = armpmu_disable, + .unthrottle = armpmu_unthrottle, + .read = armpmu_read, +}; +void hw_perf_enable (void ) {
@@ -2994,6 +3006,8 @@ init_hw_perf_events(void) perf_max_events =-1 ; } +perf_pmu_register (&pmu); +return 0 ; }arch_initcall (init_hw_perf_events);
arch/powerpc/kernel/perf_event.c +24 -22
115 lines not on this page · raw message
arch/powerpc/kernel/perf_event_fsl_emb.c +19 -18
@@ -378,13 +378,6 @@ static void fsl_emb_pmu_unthrottle(struclocal_irq_restore (flags); } -static struct pmu fsl_emb_pmu = { - .enable = fsl_emb_pmu_enable, - .disable = fsl_emb_pmu_disable, - .read = fsl_emb_pmu_read, - .unthrottle = fsl_emb_pmu_unthrottle, -}; -/* * Release the PMU if this is the last perf_event. */
@@ -428,7 +421,7 @@ static int hw_perf_cache_event(u64 confireturn 0 ; } -struct pmu *hw_perf_event_init (struct perf_event *event) +static int fsl_emb_pmu_event_init (struct perf_event *event) {u64 ev;struct perf_event *events[MAX_HWEVENTS ];
@@ -441,14 +434,14 @@ struct pmu *hw_perf_event_init(struct pecase PERF_TYPE_HARDWARE : ev = event->attr.config;if (ev >= ppmu->n_generic || ppmu->generic_events[ev] ==0 ) -return ERR_PTR (-EOPNOTSUPP ); +return -EOPNOTSUPP ; ev = ppmu->generic_events[ev];break ;case PERF_TYPE_HW_CACHE : err =hw_perf_cache_event (event->attr.config, &ev);if (err) -return ERR_PTR (err); +return err;break ;case PERF_TYPE_RAW :
@@ -456,12 +449,12 @@ struct pmu *hw_perf_event_init(struct pebreak ;default : -return ERR_PTR (-EINVAL ); +return -ENOENT ; } event->hw.config = ppmu->xlate_event (ev);if (!(event->hw.config &FSL_EMB_EVENT_VALID )) -return ERR_PTR (-EINVAL ); +return -EINVAL ;/* * If this is in a group, check if it can go on with all the
@@ -473,7 +466,7 @@ struct pmu *hw_perf_event_init(struct pe n =collect_events (event->group_leader, ppmu->n_counter -1 , events);if (n <0 ) -return ERR_PTR (-EINVAL ); +return -EINVAL ; }if (event->hw.config &FSL_EMB_EVENT_RESTRICTED ) {
@@ -484,7 +477,7 @@ struct pmu *hw_perf_event_init(struct pe }if (num_restricted >= ppmu->n_restricted) -return ERR_PTR (-EINVAL ); +return -EINVAL ; } event->hw.idx =-1 ;
@@ -497,7 +490,7 @@ struct pmu *hw_perf_event_init(struct peif (event->attr.exclude_kernel) event->hw.config_base |=PMLCA_FCS ;if (event->attr.exclude_idle) -return ERR_PTR (-ENOTSUPP ); +return -ENOTSUPP ; event->hw.last_period = event->hw.sample_period;local64_set (&event->hw.period_left, event->hw.last_period);
@@ -523,11 +516,17 @@ struct pmu *hw_perf_event_init(struct pe } event->destroy = hw_perf_event_destroy; -if (err) -return ERR_PTR (err); -return &fsl_emb_pmu; +return err; } +static struct pmu fsl_emb_pmu = { + .event_init = fsl_emb_pmu_event_init, + .enable = fsl_emb_pmu_enable, + .disable = fsl_emb_pmu_disable, + .read = fsl_emb_pmu_read, + .unthrottle = fsl_emb_pmu_unthrottle, +}; +/* * A counter has overflowed; update its count and record * things if requested. Note that interrupts are hard-disabled
@@ -651,5 +650,7 @@ int register_fsl_emb_pmu(struct fsl_emb_pr_info ("%s performance monitor hardware support registered\n" , pmu->name); +perf_pmu_register (&fsl_emb_pmu); +return 0 ; }
arch/sh/kernel/perf_event.c +24 -11
@@ -257,26 +257,38 @@ static void sh_pmu_read(struct perf_evensh_perf_event_update (event, &event->hw, event->hw.idx); } -static struct pmu pmu = { - .enable = sh_pmu_enable, - .disable = sh_pmu_disable, - .read = sh_pmu_read, -}; - -struct pmu *hw_perf_event_init (struct perf_event *event) +static int sh_pmu_event_init (struct perf_event *event) { -int err =__hw_perf_event_init (event); +int err; + +switch (event->attr.type) { +case PERF_TYPE_RAW : +case PERF_TYPE_HW_CACHE : +case PERF_TYPE_HARDWARE : + err =__hw_perf_event_init (event); +break ; + +default : +return -ENOENT ; + } +if (unlikely (err)) {if (event->destroy) event->destroy (event); -return ERR_PTR (err); } -return &pmu; +return err; } +static struct pmu pmu = { + .event_init = sh_pmu_event_init, + .enable = sh_pmu_enable, + .disable = sh_pmu_disable, + .read = sh_pmu_read, +}; +static void sh_pmu_setup (int cpu) -{ + struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);memset (cpuhw,0 ,sizeof (struct cpu_hw_events ));
@@ -325,6 +337,7 @@ int __cpuinit register_sh_pmu(struct sh_WARN_ON (pmu->num_events >MAX_HWEVENTS ); +perf_pmu_register (&pmu);perf_cpu_notifier (sh_pmu_notifier);return 0 ; }
arch/sparc/kernel/perf_event.c +16 -13
@@ -1025,7 +1025,7 @@ out:return ret; } -static int __hw_perf_event_init (struct perf_event *event) +static int sparc_pmu_event_init (struct perf_event *event) {struct perf_event_attr *attr = &event->attr;struct perf_event *evts[MAX_HWEVENTS ];
@@ -1038,17 +1038,27 @@ static int __hw_perf_event_init(struct pif (atomic_read (&nmi_active) <0 )return -ENODEV ; -if (attr->type ==PERF_TYPE_HARDWARE ) { +switch (attr->type) { +case PERF_TYPE_HARDWARE :if (attr->config >= sparc_pmu->max_events)return -EINVAL ; pmap = sparc_pmu->event_map (attr->config); - }else if (attr->type ==PERF_TYPE_HW_CACHE ) { +break ; + +case PERF_TYPE_HW_CACHE : pmap =sparc_map_cache_event (attr->config);if (IS_ERR (pmap))return PTR_ERR (pmap); - }else +break ; + +case PERF_TYPE_RAW :return -EOPNOTSUPP ; +default : +return -ENOENT ; + + } +/* We save the enable bits in the config_base. */ hwc->config_base = sparc_pmu->irq_bit;if (!attr->exclude_user)
@@ -1143,6 +1153,7 @@ static int sparc_pmu_commit_txn(struct p }static struct pmu pmu = { + .event_init = sparc_pmu_event_init, .enable = sparc_pmu_enable, .disable = sparc_pmu_disable, .read = sparc_pmu_read,
@@ -1152,15 +1163,6 @@ static struct pmu pmu = { .commit_txn = sparc_pmu_commit_txn, }; -struct pmu *hw_perf_event_init (struct perf_event *event) -{ -int err =__hw_perf_event_init (event); - -if (err) -return ERR_PTR (err); -return &pmu; -} -void perf_event_print_debug (void ) {unsigned long flags;
@@ -1280,6 +1282,7 @@ void __init init_hw_perf_events(void)/* All sparc64 PMUs currently have 2 events. */ perf_max_events =2 ; +perf_pmu_register (&pmu);register_die_notifier (&perf_event_nmi_notifier); }
arch/x86/kernel/cpu/perf_event.c +28 -17
83 lines not on this page · raw message
include/linux/perf_event.h +9 -1
@@ -561,6 +561,13 @@ struct perf_event;* struct pmu - generic performance monitoring unit */ struct pmu { +struct list_head entry; + +/* +* Should return -ENOENT when the @event doesn't match this pmu +*/ +int (*event_init) (struct perf_event *event); +int (*enable) (struct perf_event *event);void (*disable) (struct perf_event *event);int (*start) (struct perf_event *event);
@@ -847,7 +854,8 @@ struct perf_output_handle {*/ extern int perf_max_events; -extern struct pmu *hw_perf_event_init (struct perf_event *event); +extern int perf_pmu_register (struct pmu *pmu); +extern void perf_pmu_unregister (struct pmu *pmu);extern void perf_event_task_sched_in (struct task_struct *task);extern void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next);
kernel/hw_breakpoint.c +30 -5
@@ -565,6 +565,34 @@ static struct notifier_block hw_breakpoi .priority = 0x7fffffff }; +static void bp_perf_event_destroy (struct perf_event *event) +{ +release_bp_slot (event); +} + +static int hw_breakpoint_event_init (struct perf_event *bp) +{ +int err; + +if (bp->attr.type !=PERF_TYPE_BREAKPOINT ) +return -ENOENT ; + + err =register_perf_hw_breakpoint (bp); +if (err) +return err; + + bp->destroy = bp_perf_event_destroy; + +return 0 ; +} + +static struct pmu perf_breakpoint = { + .event_init = hw_breakpoint_event_init, + .enable = arch_install_hw_breakpoint, + .disable = arch_uninstall_hw_breakpoint, + .read = hw_breakpoint_pmu_read, +}; +static int __init init_hw_breakpoint (void ) {unsigned int **task_bp_pinned;
@@ -586,6 +614,8 @@ static int __init init_hw_breakpoint(voi constraints_initialized =1 ; +perf_pmu_register (&perf_breakpoint); +return register_die_notifier (&hw_breakpoint_exceptions_nb); err_alloc:
@@ -601,8 +631,3 @@ static int __init init_hw_breakpoint(voicore_initcall (init_hw_breakpoint); -struct pmu perf_ops_bp = { - .enable = arch_install_hw_breakpoint, - .disable = arch_uninstall_hw_breakpoint, - .read = hw_breakpoint_pmu_read, -};
kernel/perf_event.c +290 -298
700 lines not on this page · raw message
arch/alpha/kernel/perf_event.c +22 -15
@@ -642,34 +642,39 @@ static int __hw_perf_event_init(struct preturn 0 ; } -static struct pmu pmu = { - .enable = alpha_pmu_enable, - .disable = alpha_pmu_disable, - .read = alpha_pmu_read, - .unthrottle = alpha_pmu_unthrottle, -}; - -/* * Main entry point to initialise a HW performance event. */ -struct pmu *hw_perf_event_init (struct perf_event *event) +static int alpha_pmu_event_init (struct perf_event *event) {int err; +switch (event->attr.type) { +case PERF_TYPE_RAW : +case PERF_TYPE_HARDWARE : +case PERF_TYPE_HW_CACHE : +break ; + +default : +return -ENOENT ; + } +if (!alpha_pmu) -return ERR_PTR (-ENODEV ); +return -ENODEV ;/* Do the real initialisation work. */ err =__hw_perf_event_init (event); -if (err) -return ERR_PTR (err); - -return &pmu; +return err; } - +static struct pmu pmu = { + .event_init = alpha_pmu_event_init, + .enable = alpha_pmu_enable, + .disable = alpha_pmu_disable, + .read = alpha_pmu_read, + .unthrottle = alpha_pmu_unthrottle, +};/* * Main entry point - enable HW performance counters.
@@ -838,5 +843,7 @@ void __init init_hw_perf_events(void)/* And set up PMU specification */ alpha_pmu = &ev67_pmu; perf_max_events = alpha_pmu->num_pmcs; + +perf_pmu_register (&pmu); }
RFC PATCH 17/19 perf: Multiple task contexts
In reply to Peter ZijlstraProvide the infrastructure for multiple task contexts.
A more flexible approach would have resulted in more pointer chases in the scheduling hot-paths. This approach has the limitation of a static number of task contexts.
Since I expect most external PMUs to be system wide, or at least node wide (as per the intel uncore unit) they won't actually need a task context.
--- include/linux/perf_event.h | 1 include/linux/sched.h | 8 - kernel/perf_event.c | 344 ++++++++++++++++++++++++++++++--------------- 3 files changed, 243 insertions(+), 110 deletions(-)
include/linux/sched.h +7 -1
@@ -1161,6 +1161,12 @@ struct sched_rt_entity {struct rcu_node ; +enum perf_event_task_context { + perf_invalid_context =-1 , + perf_hw_context =0 , + perf_nr_task_contexts, +}; +struct task_struct {volatile long state;/* -1 unrunnable, 0 runnable, >0 stopped */ void *stack;
@@ -1434,7 +1440,7 @@ struct task_struct {struct futex_pi_state *pi_state_cache;#endif #ifdef CONFIG_PERF_EVENTS -struct perf_event_context *perf_event_ctxp; +struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];struct mutex perf_event_mutex;struct list_head perf_event_list;#endif
kernel/perf_event.c +235 -109
661 lines not on this page · raw message
include/linux/perf_event.h +1 -0
@@ -572,6 +572,7 @@ struct pmu {int *__percpu pmu_disable_count;struct perf_cpu_context *__percpu pmu_cpu_context; +int task_ctx_nr;/* * Fully disable/enable this PMU, can be used to protect from the PMI
RFC PATCH 4/19 perf: Unindent labels
In reply to Peter ZijlstraFixup random annoying style bits
--- kernel/perf_event.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-)
kernel/perf_event.c +24 -19
@@ -147,7 +147,7 @@ perf_lock_task_context(struct task_strucstruct perf_event_context *ctx;rcu_read_lock (); - retry: +retry: ctx =rcu_dereference (task->perf_event_ctxp);if (ctx) {/*
@@ -601,7 +601,7 @@ void perf_event_disable(struct perf_evenreturn ; } - retry: +retry:task_oncpu_function_call (task, __perf_event_disable, event);raw_spin_lock_irq (&ctx->lock);
@@ -831,7 +831,7 @@ static void __perf_install_in_context(voif (!err && !ctx->task && cpuctx->max_pertask) cpuctx->max_pertask--; - unlock: +unlock:perf_enable ();raw_spin_unlock (&ctx->lock);
@@ -904,10 +904,12 @@ static void __perf_event_mark_enabled(st event->state =PERF_EVENT_STATE_INACTIVE ; event->tstamp_enabled = ctx->time - event->total_time_enabled; -list_for_each_entry (sub, &event->sibling_list, group_entry) -if (sub->state >=PERF_EVENT_STATE_INACTIVE ) +list_for_each_entry (sub, &event->sibling_list, group_entry) { +if (sub->state >=PERF_EVENT_STATE_INACTIVE ) { sub->tstamp_enabled = ctx->time - sub->total_time_enabled; + } + } }/*
@@ -973,7 +975,7 @@ static void __perf_event_enable(void *in } } - unlock: +unlock:raw_spin_unlock (&ctx->lock); }
@@ -1014,7 +1016,7 @@ void perf_event_enable(struct perf_eventif (event->state ==PERF_EVENT_STATE_ERROR ) event->state =PERF_EVENT_STATE_OFF ; - retry: +retry:raw_spin_unlock_irq (&ctx->lock);task_oncpu_function_call (task, __perf_event_enable, event);
@@ -1034,7 +1036,7 @@ void perf_event_enable(struct perf_eventif (event->state ==PERF_EVENT_STATE_OFF )__perf_event_mark_enabled (event, ctx); - out: +out:raw_spin_unlock_irq (&ctx->lock); }
@@ -1074,17 +1076,19 @@ static void ctx_sched_out(struct perf_evif (!ctx->nr_active) goto out_enable; -if (event_type &EVENT_PINNED ) +if (event_type &EVENT_PINNED ) {list_for_each_entry (event, &ctx->pinned_groups, group_entry)group_sched_out (event, cpuctx, ctx); + } -if (event_type &EVENT_FLEXIBLE ) +if (event_type &EVENT_FLEXIBLE ) {list_for_each_entry (event, &ctx->flexible_groups, group_entry)group_sched_out (event, cpuctx, ctx); + } out_enable:perf_enable (); - out: +out:raw_spin_unlock (&ctx->lock); }
@@ -1323,9 +1327,10 @@ ctx_flexible_sched_in(struct perf_event_if (event->cpu !=-1 && event->cpu !=smp_processor_id ())continue ; -if (group_can_go_on (event, cpuctx, can_add_hw)) +if (group_can_go_on (event, cpuctx, can_add_hw)) {if (group_sched_in (event, cpuctx, ctx)) can_add_hw =0 ; + } } }
@@ -1355,7 +1360,7 @@ ctx_sched_in(struct perf_event_context *ctx_flexible_sched_in (ctx, cpuctx);perf_enable (); - out: +out:raw_spin_unlock (&ctx->lock); }
@@ -1696,7 +1701,7 @@ static void perf_event_enable_on_exec(straw_spin_unlock (&ctx->lock);perf_event_task_sched_in (task); - out: +out:local_irq_restore (flags); }
@@ -1825,7 +1830,7 @@ static struct perf_event_context *find_gif (!ptrace_may_access (task,PTRACE_MODE_READ )) goto errout; - retry: +retry: ctx =perf_lock_task_context (task, &flags);if (ctx) {unclone_ctx (ctx);
@@ -1853,7 +1858,7 @@ static struct perf_event_context *find_gput_task_struct (task);return ctx; - errout: +errout:put_task_struct (task);return ERR_PTR (err); }
@@ -3044,7 +3049,7 @@ again:if (handle->wakeup !=local_read (&buffer->wakeup))perf_output_wakeup (handle); - out: +out:preempt_enable (); }
@@ -4352,7 +4357,7 @@ static int swevent_hlist_get_cpu(structrcu_assign_pointer (cpuctx->swevent_hlist, hlist); } cpuctx->hlist_refcount++; - exit: +exit:mutex_unlock (&cpuctx->hlist_mutex);return err;
@@ -4377,7 +4382,7 @@ static int swevent_hlist_get(struct perfput_online_cpus ();return 0 ; - fail: +fail:for_each_possible_cpu (cpu) {if (cpu == failed_cpu)break ;
RFC PATCH 2/19 perf: deconstify struct pmu
In reply to Peter Zijlstrased -ie 's/const struct pmu\>/struct pmu/g' git grep -l "const struct pmu\>"
--- arch/alpha/kernel/perf_event.c | 4 ++-- arch/arm/kernel/perf_event.c | 2 +- arch/powerpc/kernel/perf_event.c | 8 ++++---- arch/powerpc/kernel/perf_event_fsl_emb.c | 2 +- arch/sh/kernel/perf_event.c | 4 ++-- arch/sparc/kernel/perf_event.c | 10 +++++----- arch/x86/kernel/cpu/perf_event.c | 14 +++++++------- include/linux/perf_event.h | 10 +++++----- kernel/perf_event.c | 26 +++++++++++++------------- 9 files changed, 40 insertions(+), 40 deletions(-)
arch/arm/kernel/perf_event.c +1 -1
@@ -491,7 +491,7 @@ __hw_perf_event_init(struct perf_event *return err; } -const struct pmu * +struct pmu *hw_perf_event_init (struct perf_event *event) {int err =0 ;
arch/powerpc/kernel/perf_event.c +4 -4
@@ -857,7 +857,7 @@ static void power_pmu_unthrottle(struct* Set the flag to make pmu::enable() not perform the * schedulability test, it will be performed at commit time */ -void power_pmu_start_txn (const struct pmu *pmu) +void power_pmu_start_txn (struct pmu *pmu) {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events);
@@ -870,7 +870,7 @@ void power_pmu_start_txn(const struct pm* Clear the flag and pmu::enable() will perform the * schedulability test. */ -void power_pmu_cancel_txn (const struct pmu *pmu) +void power_pmu_cancel_txn (struct pmu *pmu) {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events);
@@ -882,7 +882,7 @@ void power_pmu_cancel_txn(const struct p* Perform the group schedulability test as a whole * Return 0 if success */ -int power_pmu_commit_txn (const struct pmu *pmu) +int power_pmu_commit_txn (struct pmu *pmu) {struct cpu_hw_events *cpuhw;long i, n;
@@ -1014,7 +1014,7 @@ static int hw_perf_cache_event(u64 confireturn 0 ; } -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) {u64 ev;unsigned long flags;
arch/powerpc/kernel/perf_event_fsl_emb.c +1 -1
@@ -428,7 +428,7 @@ static int hw_perf_cache_event(u64 confireturn 0 ; } -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) {u64 ev;struct perf_event *events[MAX_HWEVENTS ];
arch/sh/kernel/perf_event.c +2 -2
@@ -257,13 +257,13 @@ static void sh_pmu_read(struct perf_evensh_perf_event_update (event, &event->hw, event->hw.idx); } -static const struct pmu pmu = { +static struct pmu pmu = { .enable = sh_pmu_enable, .disable = sh_pmu_disable, .read = sh_pmu_read, }; -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) {int err =__hw_perf_event_init (event);if (unlikely (err)) {
arch/sparc/kernel/perf_event.c +5 -5
@@ -1099,7 +1099,7 @@ static int __hw_perf_event_init(struct p* Set the flag to make pmu::enable() not perform the * schedulability test, it will be performed at commit time */ -static void sparc_pmu_start_txn (const struct pmu *pmu) +static void sparc_pmu_start_txn (struct pmu *pmu) {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events);
@@ -1111,7 +1111,7 @@ static void sparc_pmu_start_txn(const st* Clear the flag and pmu::enable() will perform the * schedulability test. */ -static void sparc_pmu_cancel_txn (const struct pmu *pmu) +static void sparc_pmu_cancel_txn (struct pmu *pmu) {struct cpu_hw_events *cpuhw = &__get_cpu_var (cpu_hw_events);
@@ -1123,7 +1123,7 @@ static void sparc_pmu_cancel_txn(const s* Perform the group schedulability test as a whole * Return 0 if success */ -static int sparc_pmu_commit_txn (const struct pmu *pmu) +static int sparc_pmu_commit_txn (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);int n;
@@ -1142,7 +1142,7 @@ static int sparc_pmu_commit_txn(const streturn 0 ; } -static const struct pmu pmu = { +static struct pmu pmu = { .enable = sparc_pmu_enable, .disable = sparc_pmu_disable, .read = sparc_pmu_read,
@@ -1152,7 +1152,7 @@ static const struct pmu pmu = { .commit_txn = sparc_pmu_commit_txn, }; -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) {int err =__hw_perf_event_init (event);
arch/x86/kernel/cpu/perf_event.c +7 -7
@@ -618,7 +618,7 @@ static void x86_pmu_enable_all(int added } } -static const struct pmu pmu; +static struct pmu pmu;static inline int is_x86_event (struct perf_event *event) {
@@ -1394,7 +1394,7 @@ static inline void x86_pmu_read(struct p* Set the flag to make pmu::enable() not perform the * schedulability test, it will be performed at commit time */ -static void x86_pmu_start_txn (const struct pmu *pmu) +static void x86_pmu_start_txn (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -1407,7 +1407,7 @@ static void x86_pmu_start_txn(const stru* Clear the flag and pmu::enable() will perform the * schedulability test. */ -static void x86_pmu_cancel_txn (const struct pmu *pmu) +static void x86_pmu_cancel_txn (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -1424,7 +1424,7 @@ static void x86_pmu_cancel_txn(const str* Perform the group schedulability test as a whole * Return 0 if success */ -static int x86_pmu_commit_txn (const struct pmu *pmu) +static int x86_pmu_commit_txn (struct pmu *pmu) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);int assign[X86_PMC_IDX_MAX ];
@@ -1450,7 +1450,7 @@ static int x86_pmu_commit_txn(const strureturn 0 ; } -static const struct pmu pmu = { +static struct pmu pmu = { .enable = x86_pmu_enable, .disable = x86_pmu_disable, .start = x86_pmu_start,
@@ -1536,9 +1536,9 @@ out:return ret; } -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) { -const struct pmu *tmp; +struct pmu *tmp;int err; err =__hw_perf_event_init (event);
include/linux/perf_event.h +5 -5
@@ -578,19 +578,19 @@ struct pmu {* Start the transaction, after this ->enable() doesn't need * to do schedulability tests. */ -void (*start_txn) (const struct pmu *pmu); +void (*start_txn) (struct pmu *pmu);/* * If ->start_txn() disabled the ->enable() schedulability test * then ->commit_txn() is required to perform one. On success * the transaction is closed. On error the transaction is kept * open until ->cancel_txn() is called. */ -int (*commit_txn) (const struct pmu *pmu); +int (*commit_txn) (struct pmu *pmu);/* * Will cancel the transaction, assumes ->disable() is called for * each successfull ->enable() during the transaction. */ -void (*cancel_txn) (const struct pmu *pmu); +void (*cancel_txn) (struct pmu *pmu); };/**
@@ -669,7 +669,7 @@ struct perf_event {int nr_siblings;int group_flags;struct perf_event *group_leader; -const struct pmu *pmu; +struct pmu *pmu;enum perf_event_active_state state;unsigned int attach_state;
@@ -847,7 +847,7 @@ struct perf_output_handle {*/ extern int perf_max_events; -extern const struct pmu *hw_perf_event_init (struct perf_event *event); +extern struct pmu *hw_perf_event_init (struct perf_event *event);extern void perf_event_task_sched_in (struct task_struct *task);extern void perf_event_task_sched_out (struct task_struct *task,struct task_struct *next);
kernel/perf_event.c +13 -13
@@ -75,7 +75,7 @@ static DEFINE_SPINLOCK(perf_resource_loc/* * Architecture provided APIs - weak aliases: */ -extern __weak const struct pmu *hw_perf_event_init (struct perf_event *event) +extern __weak struct pmu *hw_perf_event_init (struct perf_event *event) {return NULL ; }
@@ -673,7 +673,7 @@ group_sched_in(struct perf_event *group_ struct perf_event_context *ctx) {struct perf_event *event, *partial_group =NULL ; -const struct pmu *pmu = group_event->pmu; +struct pmu *pmu = group_event->pmu;bool txn = false;if (group_event->state ==PERF_EVENT_STATE_OFF )
@@ -4291,7 +4291,7 @@ static int perf_swevent_int(struct perf_return 0 ; } -static const struct pmu perf_ops_generic = { +static struct pmu perf_ops_generic = { .enable = perf_swevent_enable, .disable = perf_swevent_disable, .start = perf_swevent_int,
@@ -4404,7 +4404,7 @@ static void cpu_clock_perf_event_read(stcpu_clock_perf_event_update (event); } -static const struct pmu perf_ops_cpu_clock = { +static struct pmu perf_ops_cpu_clock = { .enable = cpu_clock_perf_event_enable, .disable = cpu_clock_perf_event_disable, .read = cpu_clock_perf_event_read,
@@ -4461,7 +4461,7 @@ static void task_clock_perf_event_read(stask_clock_perf_event_update (event, time); } -static const struct pmu perf_ops_task_clock = { +static struct pmu perf_ops_task_clock = { .enable = task_clock_perf_event_enable, .disable = task_clock_perf_event_disable, .read = task_clock_perf_event_read,
@@ -4575,7 +4575,7 @@ static int swevent_hlist_get(struct perf#ifdef CONFIG_EVENT_TRACING -static const struct pmu perf_ops_tracepoint = { +static struct pmu perf_ops_tracepoint = { .enable = perf_trace_enable, .disable = perf_trace_disable, .start = perf_swevent_int,
@@ -4639,7 +4639,7 @@ static void tp_perf_event_destroy(structperf_trace_destroy (event); } -static const struct pmu *tp_perf_event_init (struct perf_event *event) +static struct pmu *tp_perf_event_init (struct perf_event *event) {int err;
@@ -4686,7 +4686,7 @@ static void perf_event_free_filter(struc#else -static const struct pmu *tp_perf_event_init (struct perf_event *event) +static struct pmu *tp_perf_event_init (struct perf_event *event) {return NULL ; }
@@ -4708,7 +4708,7 @@ static void bp_perf_event_destroy(structrelease_bp_slot (event); } -static const struct pmu *bp_perf_event_init (struct perf_event *bp) +static struct pmu *bp_perf_event_init (struct perf_event *bp) {int err;
@@ -4732,7 +4732,7 @@ void perf_bp_event(struct perf_event *bp perf_swevent_add(bp, 1, 1, &sample, regs); }#else -static const struct pmu *bp_perf_event_init (struct perf_event *bp) +static struct pmu *bp_perf_event_init (struct perf_event *bp) {return NULL ; }
@@ -4754,9 +4754,9 @@ static void sw_perf_event_destroy(structswevent_hlist_put (event); } -static const struct pmu *sw_perf_event_init (struct perf_event *event) +static struct pmu *sw_perf_event_init (struct perf_event *event) { -const struct pmu *pmu =NULL ; +struct pmu *pmu =NULL ;u64 event_id = event->attr.config;/*
@@ -4818,7 +4818,7 @@ perf_event_alloc(struct perf_event_attr perf_overflow_handler_t overflow_handler, gfp_t gfpflags) { -const struct pmu *pmu; +struct pmu *pmu;struct perf_event *event;struct hw_perf_event *hwc;long err;
arch/alpha/kernel/perf_event.c +2 -2
@@ -642,7 +642,7 @@ static int __hw_perf_event_init(struct preturn 0 ; } -static const struct pmu pmu = { +static struct pmu pmu = { .enable = alpha_pmu_enable, .disable = alpha_pmu_disable, .read = alpha_pmu_read,
@@ -653,7 +653,7 @@ static const struct pmu pmu = {/* * Main entry point to initialise a HW performance event. */ -const struct pmu *hw_perf_event_init (struct perf_event *event) +struct pmu *hw_perf_event_init (struct perf_event *event) {int err;
cover perf pmu interface changes -v4
These patches prepare the perf code for multiple PMUs.
About the first half of these patches deals with removing all the weak functions, the second half provides per pmu contexts.
New to the first half is Alpha, which got merged into mainline recently.
The second half is completely new and lightly tested, one known breakage is that it currently doesn't allow to mix software and hardware events into a single group (should be fixed soon).
Patches also available in git format for easy testing (tip/master + patches):
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/linux-2.6-perf.git perf-pmu
Head should be baf996bc6d6
---
arch/alpha/kernel/perf_event.c | 113 ++- arch/arm/kernel/perf_event.c | 136 ++- arch/powerpc/kernel/perf_event.c | 164 ++- arch/powerpc/kernel/perf_event_fsl_emb.c | 148 ++- arch/sh/kernel/perf_event.c | 143 ++- arch/sparc/kernel/perf_event.c | 164 ++- arch/x86/kernel/cpu/perf_event.c | 176 ++- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- arch/x86/kernel/cpu/perf_event_intel_ds.c | 2 +- include/linux/ftrace_event.h | 4 +- include/linux/perf_event.h | 113 +- include/linux/sched.h | 9 +- kernel/hw_breakpoint.c | 62 +- kernel/perf_event.c | 2068 +++++++++++++++-------------- kernel/sched.c | 2 - kernel/trace/trace_event_perf.c | 7 +- 16 files changed, 1889 insertions(+), 1424 deletions(-)
RFC PATCH 10/19 perf: Remove the sysfs bits
Neither the overcommit nor the reservation sysfs parameter were actually working, remove them as they'll only get in the way.
--- arch/alpha/kernel/perf_event.c | 3 arch/arm/kernel/perf_event.c | 9 -- arch/sparc/kernel/perf_event.c | 9 -- arch/x86/kernel/cpu/perf_event.c | 1 include/linux/perf_event.h | 6 - kernel/perf_event.c | 124 --------------------------------------- 6 files changed, 5 insertions(+), 147 deletions(-)
arch/alpha/kernel/perf_event.c +1 -2
@@ -808,7 +808,7 @@ static void alpha_perf_event_irq_handlerwrperfmon (PERFMON_CMD_DISABLE , cpuc->idx_mask);/* la_ptr is the counter that overflowed. */ -if (unlikely (la_ptr >= perf_max_events)) { +if (unlikely (la_ptr >= alpha_pmu->num_pmcs)) {/* This should never occur! */ irq_err_count++;pr_warning ("PMI: silly index %ld\n" , la_ptr);
@@ -879,7 +879,6 @@ void __init init_hw_perf_events(void)/* And set up PMU specification */ alpha_pmu = &ev67_pmu; - perf_max_events = alpha_pmu->num_pmcs;perf_pmu_register (&pmu); }
arch/arm/kernel/perf_event.c +1 -8
@@ -534,7 +534,7 @@ static int armpmu_event_init(struct perf event->destroy = hw_perf_event_destroy;if (!atomic_inc_not_zero (&active_events)) { -if (atomic_read (&active_events) > perf_max_events) { +if (atomic_read (&active_events) > armpmu.num_events) {atomic_dec (&active_events);return -ENOSPC ; }
@@ -2974,14 +2974,12 @@ init_hw_perf_events(void) armpmu = &armv6pmu;memcpy (armpmu_perf_cache_map, armv6_perf_cache_map,sizeof (armv6_perf_cache_map)); - perf_max_events = armv6pmu.num_events;break ;case 0xB020 :/* ARM11mpcore */ armpmu = &armv6mpcore_pmu;memcpy (armpmu_perf_cache_map, armv6mpcore_perf_cache_map,sizeof (armv6mpcore_perf_cache_map)); - perf_max_events = armv6mpcore_pmu.num_events;break ;case 0xC080 :/* Cortex-A8 */ armv7pmu.id =ARM_PERF_PMU_ID_CA8 ;
@@ -2993,7 +2991,6 @@ init_hw_perf_events(void)/* Reset PMNC and read the nb of CNTx counters supported */ armv7pmu.num_events =armv7_reset_read_pmnc (); - perf_max_events = armv7pmu.num_events;break ;case 0xC090 :/* Cortex-A9 */ armv7pmu.id =ARM_PERF_PMU_ID_CA9 ;
@@ -3005,7 +3002,6 @@ init_hw_perf_events(void)/* Reset PMNC and read the nb of CNTx counters supported */ armv7pmu.num_events =armv7_reset_read_pmnc (); - perf_max_events = armv7pmu.num_events;break ; }/* Intel CPUs [xscale]. */
@@ -3016,13 +3012,11 @@ init_hw_perf_events(void) armpmu = &xscale1pmu;memcpy (armpmu_perf_cache_map, xscale_perf_cache_map,sizeof (xscale_perf_cache_map)); - perf_max_events = xscale1pmu.num_events;break ;case 2 : armpmu = &xscale2pmu;memcpy (armpmu_perf_cache_map, xscale_perf_cache_map,sizeof (xscale_perf_cache_map)); - perf_max_events = xscale2pmu.num_events;break ; } }
@@ -3032,7 +3026,6 @@ init_hw_perf_events(void) arm_pmu_names[armpmu->id], armpmu->num_events); }else {pr_info ("no hardware support available\n" ); - perf_max_events =-1 ; }perf_pmu_register (&pmu);
arch/sparc/kernel/perf_event.c +3 -6
@@ -897,7 +897,7 @@ static int sparc_check_constraints(strucif (!n_ev)return 0 ; -if (n_ev > perf_max_events) +if (n_ev >MAX_HWEVENTS )return -1 ; msk0 =perf_event_get_msk (events[0 ]);
@@ -1014,7 +1014,7 @@ static int sparc_pmu_add(struct perf_eveperf_pmu_disable (event->pmu); n0 = cpuc->n_events; -if (n0 >= perf_max_events) +if (n0 >=MAX_HWEVENTS ) goto out; cpuc->event[n0] = event;
@@ -1097,7 +1097,7 @@ static int sparc_pmu_event_init(struct p n =0 ;if (event->group_leader != event) { n =collect_events (event->group_leader, - perf_max_events -1 , +MAX_HWEVENTS -1 , evts, events, current_idx_dmy);if (n <0 )return -EINVAL ;
@@ -1309,9 +1309,6 @@ void __init init_hw_perf_events(void)pr_cont ("Supported PMU type is '%s'\n" , sparc_pmu_type); -/* All sparc64 PMUs currently have 2 events. */ - perf_max_events =2 ; -perf_pmu_register (&pmu);register_die_notifier (&perf_event_nmi_notifier); }
arch/x86/kernel/cpu/perf_event.c +0 -1
@@ -1395,7 +1395,6 @@ void __init init_hw_perf_events(void) x86_pmu.num_counters =X86_PMC_MAX_GENERIC ; } x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) -1 ; - perf_max_events = x86_pmu.num_counters;if (x86_pmu.num_counters_fixed >X86_PMC_MAX_FIXED ) { WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
include/linux/perf_event.h +0 -6
@@ -860,7 +860,6 @@ struct perf_cpu_context {struct perf_event_context ctx;struct perf_event_context *task_ctx;int active_oncpu; -int max_pertask;int exclusive;struct swevent_hlist *swevent_hlist;struct mutex hlist_mutex;
@@ -883,11 +882,6 @@ struct perf_output_handle {#ifdef CONFIG_PERF_EVENTS -/* -* Set by architecture code: -*/ -extern int perf_max_events; -extern int perf_pmu_register (struct pmu *pmu);extern void perf_pmu_unregister (struct pmu *pmu);
kernel/perf_event.c +0 -124
157 lines not on this page · raw message
Peter Zijlstra
maintainerRe: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
(quoted diff) +4
-0@@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c cpuctx = &per_cpu(perf_cpu_context, cpu); __perf_event_init_context(&cpuctx->ctx, NULL); + cpuctx->timer_interval = TICK_NSEC; + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + cpuctx->timer.function = perf_event_context_tick; } +static void perf_pmu_rotate_start(struct pmu *pmu)
Review static void __init perf_event_init_all_c
Quoted 10 lines
(quoted diff) +9
-0+{ + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); + + if (hrtimer_active(&cpuctx->timer)) + return; + + __hrtimer_start_range_ns(&cpuctx->timer, + ns_to_ktime(cpuctx->timer_interval), 0, + HRTIMER_MODE_REL, 0); }
This probably wants a fuzz factor that lets it fold into the tick we already have. Thomas what's the easiest way to do that, give it a soft limit of 1ns and hardlimit of TICK_NSEC?
Peter Zijlstra
maintainerRe: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
In reply to Peter ZijlstraQuoted 12 lines · Peter Zijlstra
+static enum
hrtimer_restartperf_event_context_tick(struct hrtimer *timer)
{kernel/perf_event.c +4
-0+enum hrtimer_restart restart =HRTIMER_NORESTART ;struct perf_cpu_context *cpuctx;struct perf_event_context *ctx;int rotate =0 ; + cpuctx = container_of(timer, struct perf_cpu_context, timer); +if (cpuctx->ctx.nr_events) { + restart =HRTIMER_RESTART ;
kernel/perf_event.c static void rotate_ctx(struct perf_event · line 1567
Quoted 10 lines
kernel/perf_event.c +9
-0+if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) + rotate =1 ; + } + ctx = current->perf_event_ctxp; +if (ctx && ctx->nr_events) { + restart =HRTIMER_RESTART ; +if (ctx->nr_events != ctx->nr_active) + rotate =1 ; + }
Note to self, this is probably racy against pmu_rotate_start().
Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
In reply to Peter ZijlstraQuoted 11 lines · Peter Zijlstra
Quoted 8 lines · Peter Zijlstra
kernel/perf_event.c +3
-0@@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c cpuctx = &per_cpu (perf_cpu_context, cpu);__perf_event_init_context (&cpuctx->ctx,NULL ); + cpuctx->timer_interval =TICK_NSEC ; +hrtimer_init (&cpuctx->timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + cpuctx->timer.function = perf_event_context_tick; }
Quoted 15 lines
kernel/perf_event.c void perf_event_task_tick(struct task_st · line 1604
Quoted 11 lines
kernel/perf_event.c +10
-0+static void perf_pmu_rotate_start (struct pmu *pmu) +{ +struct perf_cpu_context *cpuctx = &__get_cpu_var (perf_cpu_context); + +if (hrtimer_active (&cpuctx->timer)) +return ; + +__hrtimer_start_range_ns (&cpuctx->timer, +ns_to_ktime (cpuctx->timer_interval),0 , +HRTIMER_MODE_REL ,0 ); }This probably wants a fuzz factor that lets it fold into the tick we already have. Thomas what's the easiest way to do that, give it a soft limit of 1ns and hardlimit of
TICK_NSEC?
Hmm, why don't you hang it off the tick right away ?
Thanks,
tglx
Peter Zijlstra
maintainerRe: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
Quoted 29 lines · Thomas Gleixner
Quoted 11 lines · Peter Zijlstra
Quoted 8 lines · Peter Zijlstra
kernel/perf_event.c +3
-0@@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c cpuctx = &per_cpu (perf_cpu_context, cpu);__perf_event_init_context (&cpuctx->ctx,NULL ); + cpuctx->timer_interval =TICK_NSEC ; +hrtimer_init (&cpuctx->timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + cpuctx->timer.function = perf_event_context_tick; }Quoted 15 lines
kernel/perf_event.c void perf_event_task_tick(struct task_st · line 1604
Quoted 11 lines
kernel/perf_event.c +10
-0+static void perf_pmu_rotate_start (struct pmu *pmu) +{ +struct perf_cpu_context *cpuctx = &__get_cpu_var (perf_cpu_context); + +if (hrtimer_active (&cpuctx->timer)) +return ; + +__hrtimer_start_range_ns (&cpuctx->timer, +ns_to_ktime (cpuctx->timer_interval),0 , +HRTIMER_MODE_REL ,0 ); }This probably wants a fuzz factor that lets it fold into the tick we already have. Thomas what's the easiest way to do that, give it a soft limit of 1ns and hardlimit of
TICK_NSEC?
Quoted 1 line
Hmm, why don't you hang it off the tick right away ?
Because some people (Corey) want to be able to have different rotation periods per pmu driver, so a timer per context was the easy way out.
Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
Quoted 30 lines · Peter Zijlstra
Quoted 29 lines · Thomas Gleixner
Quoted 26 lines · Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
(quoted diff) +4
-0@@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c cpuctx = &per_cpu(perf_cpu_context, cpu); __perf_event_init_context(&cpuctx->ctx, NULL); + cpuctx->timer_interval = TICK_NSEC; + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + cpuctx->timer.function = perf_event_context_tick; } +static void perf_pmu_rotate_start(struct pmu *pmu)Review static void __init perf_event_init_all_c
Quoted 10 lines
(quoted diff) +9
-0+{ + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); + + if (hrtimer_active(&cpuctx->timer)) + return; + + __hrtimer_start_range_ns(&cpuctx->timer, + ns_to_ktime(cpuctx->timer_interval), 0, + HRTIMER_MODE_REL, 0); }This probably wants a fuzz factor that lets it fold into the tick we already have. Thomas what's the easiest way to do that, give it a soft limit of 1ns and hardlimit of
TICK_NSEC?
Quoted 4 lines
Quoted 1 line
Hmm, why don't you hang it off the tick right away ?
Because some people (Corey) want to be able to have different rotation periods per pmu driver, so a timer per context was the easy way out.
Makes sense. If you want it fully aligned with the tick, then you just might get the real tick time from the tick code and make it proper aligned.
Thanks,
tglx
Peter Zijlstra
maintainerRe: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
Quoted 6 lines · Thomas Gleixner
Quoted 2 lines
Because some people (Corey) want to be able to have different rotation periods per pmu driver, so a timer per context was the easy way out.
Makes sense. If you want it fully aligned with the tick, then you just might get the real tick time from the tick code and make it proper aligned.
Right, I'll go see if I can make that work nicely.
Quoted 10 lines · Peter Zijlstra
These patches prepare the perf code for multiple PMUs.
About the first half of these patches deals with removing all the weak functions, the second half provides per pmu contexts.
New to the first half is Alpha, which got merged into mainline recently.
The second half is completely new and lightly tested, one known breakage is that it currently doesn't allow to mix software and hardware events into a single group (should be fixed soon).
I see Ingo committed this series into the tip perf/core branch. Did this problem get fixed before that, or does it still need to be fixed?
Paul.
Peter Zijlstra
maintainerQuoted 5 lines · Paul Mackerras
Quoted 2 lines
that it currently doesn't allow to mix software and hardware events into a single group (should be fixed soon).
I see Ingo committed this series into the tip perf/core branch. Did this problem get fixed before that, or does it still need to be fixed?
I thought I fixed it, but I realized I only fixed half of it.
The current state is that it allows to add software events to hardware groups, but does not allow adding a hardware event to a pure software group.
I'll try and get a patch done that attempts fixing that.
Peter Zijlstra
maintainerQuoted 7 lines · Paul Mackerras
Quoted 5 lines · Peter Zijlstra
These patches prepare the perf code for multiple PMUs.
About the first half of these patches deals with removing all the weak functions, the second half provides per pmu contexts.
Quoted 8 lines
Quoted 5 lines
New to the first half is Alpha, which got merged into mainline recently.
The second half is completely new and lightly tested, one known breakage is that it currently doesn't allow to mix software and hardware events into a single group (should be fixed soon).
I see Ingo committed this series into the tip perf/core branch. Did this problem get fixed before that, or does it still need to be fixed?
I was thinking something like the below, except I obviously need to sort that XXX thing.
--- kernel/perf_event.c | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-)
kernel/perf_event.c +40 -8
@@ -494,7 +494,7 @@ static void __perf_event_remove_from_conraw_spin_lock (&ctx->lock); -event_sched_out (event, cpuctx, ctx); +group_sched_out (event, cpuctx, ctx);list_del_event (event, ctx);
@@ -5522,9 +5522,10 @@ SYSCALL_DEFINE5(perf_event_open,struct perf_event_context *ctx;struct file *event_file =NULL ;struct file *group_file =NULL ; -struct task_struct *task; +struct task_struct *task =NULL ;struct pmu *pmu;int event_fd; +int move_group_leader =0 ;int fput_needed =0 ;int err;
@@ -5574,15 +5575,34 @@ SYSCALL_DEFINE5(perf_event_open,* any hardware group. */ pmu = event->pmu; -if ((pmu->task_ctx_nr == perf_sw_context) && group_leader) - pmu = group_leader->pmu; + +if (group_leader && + (is_software_event (event) !=is_software_event (group_leader))) { +if (is_software_event (event)) { +/* +* If event and group_leader are not both a software +* event, and event it, then group leader is not. +* +* Allow the addition of software events to !software +* groups, this is safe because software events never +* fail to schedule. +*/ + pmu = group_leader->pmu; + }else if (is_software_event (group_leader) && + (group_leader->group_flags &PERF_GROUP_SOFTWARE )) { +/* +* In case the group is a pure software group, and we +* try to add a hardware event, move the whole group to +* the hardware context. +*/ + move_group_leader =1 ; + } + }/* * Get the target context (task or percpu): */ -if (pid ==-1 && cpu !=-1 ) - task =NULL ; -else +if (!(pid ==-1 && cpu !=-1 )) task =find_lively_task_by_vpid (pid); ctx =find_get_context (pmu, task, cpu);
@@ -5606,8 +5626,10 @@ SYSCALL_DEFINE5(perf_event_open,/* * Do not allow to attach to a group in a different * task or CPU context: +* +* XXX fix the task vs cpu context thing for move_group_leader */ -if (group_leader->ctx != ctx) +if (!move_group_leader && group_leader->ctx != ctx) goto err_context;/* * Only a group leader can be exclusive or pinned
@@ -5628,9 +5650,19 @@ SYSCALL_DEFINE5(perf_event_open, goto err_context; } +if (move_group_leader) { +struct perf_event_context *gctx = group_leader->ctx; + +mutex_lock (&gctx->mutex); +perf_event_remove_from_context (group_leader); +mutex_unlock (&gctx->mutex); + } + event->filp = event_file;WARN_ON_ONCE (ctx->parent_ctx);mutex_lock (&ctx->mutex); +if (move_group_leader) +perf_install_in_context (ctx, group_leader, cpu);perf_install_in_context (ctx, event, cpu); ++ctx->generation;mutex_unlock (&ctx->mutex);