perf: Convert perf_event to local_t
Subsystems (7)
Commit-ID: e78505958cf123048fb48cb56b79cebb8edd15fb Gitweb: http://git.kernel.org/tip/e78505958cf123048fb48cb56b79cebb8edd15fb Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 21 May 2010 14:43:08 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 9 Jun 2010 11:12:37 +0200
perf: Convert perf_event to local_t
Since now all modification to event->count (and ->prev_count and ->period_left) are local to a cpu, change then to local64_t so we avoid the LOCK'ed ops.
Signed-off-by Peter Zijlstra <[email protected]>
LKML-Reference: <new-submission>
Signed-off-by Ingo Molnar <[email protected]>
--- arch/arm/kernel/perf_event.c | 18 ++++++++-------- arch/powerpc/kernel/perf_event.c | 34 +++++++++++++++--------------- arch/sh/kernel/perf_event.c | 6 ++-- arch/sparc/kernel/perf_event.c | 18 ++++++++-------- arch/x86/kernel/cpu/perf_event.c | 18 ++++++++-------- include/linux/perf_event.h | 7 +++-- kernel/perf_event.c | 42 +++++++++++++++++++------------------- 7 files changed, 72 insertions(+), 71 deletions(-)
arch/arm/kernel/perf_event.c +9 -9
@@ -164,20 +164,20 @@ armpmu_event_set_period(struct perf_event *event, struct hw_perf_event *hwc, int idx) { -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 ;if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }
@@ -185,7 +185,7 @@ armpmu_event_set_period(struct perf_event *event,if (left > (s64 )armpmu->max_period) left = armpmu->max_period; -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left); armpmu->write_counter (idx, (u64)(-left) &0xffffffff );
@@ -204,18 +204,18 @@ armpmu_event_update(struct perf_event *event,s64 delta; again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count = armpmu->read_counter (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again; delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -478,7 +478,7 @@ __hw_perf_event_init(struct perf_event *event)if (!hwc->sample_period) { hwc->sample_period = armpmu->max_period; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); } err =0 ;
arch/powerpc/kernel/perf_event.c +17 -17
@@ -410,15 +410,15 @@ static void power_pmu_read(struct perf_event *event)* Therefore we treat them like NMIs. */ do { - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count);barrier (); val =read_pmc (event->hw.idx); - }while (atomic64_cmpxchg (&event->hw.prev_count, prev, val) != prev); + }while (local64_cmpxchg (&event->hw.prev_count, prev, val) != prev);/* The counters are only 32 bits wide */ delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &event->hw.period_left); +local64_add (delta, &event->count); +local64_sub (delta, &event->hw.period_left); }/*
@@ -444,10 +444,10 @@ static void freeze_limited_counters(struct cpu_hw_events *cpuhw,if (!event->hw.idx)continue ; val = (event->hw.idx ==5 ) ? pmc5 : pmc6; - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count); event->hw.idx =0 ; delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count); } }
@@ -462,7 +462,7 @@ static void thaw_limited_counters(struct cpu_hw_events *cpuhw, event = cpuhw->limited_counter[i]; event->hw.idx = cpuhw->limited_hwidx[i]; val = (event->hw.idx ==5 ) ? pmc5 : pmc6; -atomic64_set (&event->hw.prev_count, val); +local64_set (&event->hw.prev_count, val);perf_event_update_userpage (event); } }
@@ -666,11 +666,11 @@ void hw_perf_enable(void) } val =0 ;if (event->hw.sample_period) { - left =atomic64_read (&event->hw.period_left); + left =local64_read (&event->hw.period_left);if (left <0x80000000L ) val =0x80000000L - left; } -atomic64_set (&event->hw.prev_count, val); +local64_set (&event->hw.prev_count, val); event->hw.idx = idx;write_pmc (idx, val);perf_event_update_userpage (event);
@@ -842,8 +842,8 @@ static void power_pmu_unthrottle(struct perf_event *event)if (left <0x80000000L ) val =0x80000000L - left;write_pmc (event->hw.idx, val); -atomic64_set (&event->hw.prev_count, val); -atomic64_set (&event->hw.period_left, left); +local64_set (&event->hw.prev_count, val); +local64_set (&event->hw.period_left, left);perf_event_update_userpage (event);perf_enable ();local_irq_restore (flags);
@@ -1109,7 +1109,7 @@ const struct pmu *hw_perf_event_init(struct perf_event *event) event->hw.config = events[n]; event->hw.event_base = cflags[n]; event->hw.last_period = event->hw.sample_period; -atomic64_set (&event->hw.period_left, event->hw.last_period); +local64_set (&event->hw.period_left, event->hw.last_period);/* * See if we need to reserve the PMU.
@@ -1147,16 +1147,16 @@ static void record_and_restart(struct perf_event *event, unsigned long val,int record =0 ;/* we don't have to worry about interrupts here */ - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count); delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count);/* * See if the total period for this event has expired, * and update for the next period. */ val =0 ; - left =atomic64_read (&event->hw.period_left) - delta; + left =local64_read (&event->hw.period_left) - delta;if (period) {if (left <=0 ) { left += period;
@@ -1194,8 +1194,8 @@ static void record_and_restart(struct perf_event *event, unsigned long val, }write_pmc (event->hw.idx, val); -atomic64_set (&event->hw.prev_count, val); -atomic64_set (&event->hw.period_left, left); +local64_set (&event->hw.prev_count, val); +local64_set (&event->hw.period_left, left);perf_event_update_userpage (event); }
arch/sh/kernel/perf_event.c +3 -3
@@ -185,10 +185,10 @@ static void sh_perf_event_update(struct perf_event *event,* this is the simplest approach for maintaining consistency. */ again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count = sh_pmu->read (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again;
@@ -203,7 +203,7 @@ again: delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count); }static void sh_pmu_disable (struct perf_event *event)
arch/sparc/kernel/perf_event.c +9 -9
@@ -572,18 +572,18 @@ static u64 sparc_perf_event_update(struct perf_event *event,s64 delta; again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count =read_pmc (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again; delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -591,27 +591,27 @@ again:static int sparc_perf_event_set_period (struct perf_event *event,struct hw_perf_event *hwc,int idx) { -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 ;if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (left >MAX_PERIOD ) left =MAX_PERIOD ; -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left);write_pmc (idx, (u64)(-left) &0xffffffff );
@@ -1087,7 +1087,7 @@ static int __hw_perf_event_init(struct perf_event *event)if (!hwc->sample_period) { hwc->sample_period =MAX_PERIOD ; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); }return 0 ;
arch/x86/kernel/cpu/perf_event.c +9 -9
@@ -296,10 +296,10 @@ x86_perf_event_update(struct perf_event *event)* count to the generic event atomically: */ again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count);rdmsrl (hwc->event_base + idx, new_raw_count); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again;
@@ -314,8 +314,8 @@ again: delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -439,7 +439,7 @@ static int x86_setup_perfctr(struct perf_event *event)if (!hwc->sample_period) { hwc->sample_period = x86_pmu.max_period; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); }else {/* * If we have a PMU initialized but no APIC
@@ -886,7 +886,7 @@ static int x86_perf_event_set_period(struct perf_event *event) {struct hw_perf_event *hwc = &event->hw; -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 , idx = hwc->idx;
@@ -898,14 +898,14 @@ x86_perf_event_set_period(struct perf_event *event)*/ if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }
@@ -924,7 +924,7 @@ x86_perf_event_set_period(struct perf_event *event)* The hw event starts counting from this event offset, * mark it to be able to extra future deltas: */ -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left);wrmsrl (hwc->event_base + idx, (u64)(-left) & x86_pmu.cntval_mask);
include/linux/perf_event.h +4 -3
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {#include <linux/cpu.h> #include <asm/atomic.h> #include <asm/local.h> +#include <asm/local64.h> #define PERF_MAX_STACK_DEPTH 255
@@ -536,10 +537,10 @@ struct hw_perf_event {struct arch_hw_breakpoint info;#endif }; -atomic64_t prev_count; +local64_t prev_count;u64 sample_period;u64 last_period; -atomic64_t period_left; +local64_t period_left;u64 interrupts;u64 freq_time_stamp;
@@ -670,7 +671,7 @@ struct perf_event {enum perf_event_active_state state;unsigned int attach_state; -atomic64_t count; +local64_t count;atomic64_t child_count;/*
kernel/perf_event.c +21 -21
141 lines not on this page · raw message