convert perf to local64_t
Subsystems (15)
cover convert perf to local64_t
These patches introduce local64_t.
Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use local64_t instead of atomic64_t and use instructions without buslock semantics.
The local64_t implementation uses local_t for 64 bits, since local_t is of type long, for 32 bit it falls back to atomic64_t. Architectures can provide their own implementation as usual.
PATCH 1/4 arch: local64_t
Implements local64_t.
On 64bit, local_t is of size long, and thus we make local64_t an alias. On 32bit, we fall back to atomic64_t.
LKML-Reference: <new-submission>
--- arch/alpha/include/asm/local64.h | 1 arch/arm/include/asm/local64.h | 1 arch/avr32/include/asm/local64.h | 1 arch/blackfin/include/asm/local64.h | 1 arch/cris/include/asm/local64.h | 1 arch/frv/include/asm/local64.h | 1 arch/frv/kernel/local64.h | 1 arch/h8300/include/asm/local64.h | 1 arch/ia64/include/asm/local64.h | 1 arch/m32r/include/asm/local64.h | 1 arch/m68k/include/asm/local64.h | 1 arch/microblaze/include/asm/local64.h | 1 arch/mips/include/asm/local64.h | 1 arch/mn10300/include/asm/local64.h | 1 arch/parisc/include/asm/local64.h | 1 arch/powerpc/include/asm/local64.h | 1 arch/s390/include/asm/local64.h | 1 arch/score/include/asm/local64.h | 1 arch/sh/include/asm/local64.h | 1 arch/sparc/include/asm/local64.h | 1 arch/x86/include/asm/local64.h | 1 arch/xtensa/include/asm/local64.h | 1 include/asm-generic/local64.h | 96 ++++++++++++++++++++++++++++++++++ 23 files changed, 118 insertions(+)
arch/alpha/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/arm/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/avr32/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/blackfin/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/cris/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/frv/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/frv/kernel/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/h8300/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/ia64/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/m32r/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/m68k/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/microblaze/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/mips/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/mn10300/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/parisc/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/powerpc/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/s390/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/score/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/sh/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/sparc/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/x86/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
arch/xtensa/include/asm/local64.h new file +1 -0
@@ -0,0 +1 @@ +#include <asm-generic/local64.h>
include/asm-generic/local64.h new file +96 -0
96 lines not on this page · raw message
Re: [PATCH 1/4] arch: local64_t
Quoted 8 lines · Peter Zijlstra
Implements
local64_t.On 64bit,
local_tis of size long, and thus we makelocal64_tan alias. On 32bit, we fall back toatomic64_t.Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
Header looks good.
Re: [PATCH 1/4] arch: local64_t
In reply to Peter ZijlstraQuoted 4 lines · Peter Zijlstra
+ * This is the default implementation, which uses
atomic64_t. Which is
+ * rather pointless. The whole point behindlocal64_tis that some processors
+ * can perform atomic adds and subtracts in a manner which is atomic wrt IRQs
+ * running on this CPU.local64_tallows exploitation of such capabilities.
Interesting... What FRV does in atomic64-ops.S should probably be rebranded local64_t, and atomic64_t ops be based on that in non-SMP mode.
What I did on FRV was to emulate LL/ST instructions using some of the excessive numbers of conditional bits to do so - but it only works on UP systems.
David
PATCH 2/4 perf: Add perf_event_count()
In reply to Peter ZijlstraCreate a helper function for those sites that want to read the event count.
LKML-Reference: <new-submission>
--- kernel/perf_event.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)
kernel/perf_event.c +11 -6
@@ -1701,6 +1701,11 @@ static void __perf_event_read(void *info event->pmu->read (event); } +static inline u64 perf_event_count (struct perf_event *event) +{ +return atomic64_read (&event->count); +} +static u64 perf_event_read (struct perf_event *event) {/*
@@ -1720,7 +1725,7 @@ static u64 perf_event_read(struct perf_eraw_spin_unlock_irqrestore (&ctx->lock, flags); } -return atomic64_read (&event->count); +return perf_event_count (event); }/*
@@ -2280,7 +2285,7 @@ void perf_event_update_userpage(struct p ++userpg->lock;barrier (); userpg->index =perf_event_index (event); - userpg->offset =atomic64_read (&event->count); + userpg->offset =perf_event_count (event);if (event->state ==PERF_EVENT_STATE_ACTIVE ) userpg->offset -=atomic64_read (&event->hw.prev_count);
@@ -3125,7 +3130,7 @@ static void perf_output_read_one(structu64 values[4 ];int n =0 ; - values[n++] =atomic64_read (&event->count); + values[n++] =perf_event_count (event);if (read_format &PERF_FORMAT_TOTAL_TIME_ENABLED ) { values[n++] = event->total_time_enabled +atomic64_read (&event->child_total_time_enabled);
@@ -3162,7 +3167,7 @@ static void perf_output_read_group(strucif (leader != event) leader->pmu->read (leader); - values[n++] =atomic64_read (&leader->count); + values[n++] =perf_event_count (leader);if (read_format &PERF_FORMAT_ID ) values[n++] =primary_event_id (leader);
@@ -3174,7 +3179,7 @@ static void perf_output_read_group(strucif (sub != event) sub->pmu->read (sub); - values[n++] =atomic64_read (&sub->count); + values[n++] =perf_event_count (sub);if (read_format &PERF_FORMAT_ID ) values[n++] =primary_event_id (sub);
@@ -5272,7 +5277,7 @@ static void sync_child_event(struct perfif (child_event->attr.inherit_stat)perf_event_read_event (child_event, child); - child_val =atomic64_read (&child_event->count); + child_val =perf_event_count (child_event);/* * Add back the child's count to the parent's count:
PATCH 3/4 perf: Add child_count
In reply to Peter ZijlstraOnly child counters adding back their values into the parent counter are responsible for cross-cpu updates to event->count.
So if we pull that out into a new child_count variable, we get an event->count that is only modified locally.
LKML-Reference: <new-submission>
--- include/linux/perf_event.h | 1 + kernel/perf_event.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-)
include/linux/perf_event.h +1 -0
@@ -648,6 +648,7 @@ struct perf_event {enum perf_event_active_state state;atomic64_t count; +atomic64_t child_count;/* * These are the total time in nanoseconds that the event
kernel/perf_event.c +2 -2
@@ -1703,7 +1703,7 @@ static void __perf_event_read(void *infostatic inline u64 perf_event_count (struct perf_event *event) { -return atomic64_read (&event->count); +return atomic64_read (&event->count) +atomic64_read (&event->child_count); }static u64 perf_event_read (struct perf_event *event)
@@ -5282,7 +5282,7 @@ static void sync_child_event(struct perf/* * Add back the child's count to the parent's count: */ -atomic64_add (child_val, &parent_event->count); +atomic64_add (child_val, &parent_event->child_count);atomic64_add (child_event->total_time_enabled, &parent_event->child_total_time_enabled); atomic64_add(child_event->total_time_running,
PATCH 4/4 perf: Convert perf_event to local_t
In reply to Peter ZijlstraSince now all modification to event->count (and ->prev_count and ->period_left) are local to a cpu, change then to local64_t so we avoid the LOCK'ed ops.
LKML-Reference: <new-submission>
--- arch/arm/kernel/perf_event.c | 18 ++++++++-------- arch/powerpc/kernel/perf_event.c | 34 +++++++++++++++---------------- arch/sh/kernel/perf_event.c | 6 ++--- arch/sparc/kernel/perf_event.c | 18 ++++++++-------- arch/x86/kernel/cpu/perf_event.c | 18 ++++++++-------- include/linux/perf_event.h | 7 +++--- kernel/perf_event.c | 42 +++++++++++++++++++-------------------- 7 files changed, 72 insertions(+), 71 deletions(-)
arch/arm/kernel/perf_event.c +9 -9
@@ -128,20 +128,20 @@ armpmu_event_set_period(struct perf_even struct hw_perf_event *hwc, int idx) { -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 ;if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }
@@ -149,7 +149,7 @@ armpmu_event_set_period(struct perf_evenif (left > (s64 )armpmu->max_period) left = armpmu->max_period; -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left); armpmu->write_counter (idx, (u64)(-left) &0xffffffff );
@@ -168,18 +168,18 @@ armpmu_event_update(struct perf_event *es64 delta; again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count = armpmu->read_counter (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again; delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -433,7 +433,7 @@ __hw_perf_event_init(struct perf_event *if (!hwc->sample_period) { hwc->sample_period = armpmu->max_period; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); } err =0 ;
arch/powerpc/kernel/perf_event.c +17 -17
@@ -410,15 +410,15 @@ static void power_pmu_read(struct perf_e* Therefore we treat them like NMIs. */ do { - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count);barrier (); val =read_pmc (event->hw.idx); - }while (atomic64_cmpxchg (&event->hw.prev_count, prev, val) != prev); + }while (local64_cmpxchg (&event->hw.prev_count, prev, val) != prev);/* The counters are only 32 bits wide */ delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &event->hw.period_left); +local64_add (delta, &event->count); +local64_sub (delta, &event->hw.period_left); }/*
@@ -444,10 +444,10 @@ static void freeze_limited_counters(struif (!event->hw.idx)continue ; val = (event->hw.idx ==5 ) ? pmc5 : pmc6; - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count); event->hw.idx =0 ; delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count); } }
@@ -462,7 +462,7 @@ static void thaw_limited_counters(struct event = cpuhw->limited_counter[i]; event->hw.idx = cpuhw->limited_hwidx[i]; val = (event->hw.idx ==5 ) ? pmc5 : pmc6; -atomic64_set (&event->hw.prev_count, val); +local64_set (&event->hw.prev_count, val);perf_event_update_userpage (event); } }
@@ -666,11 +666,11 @@ void hw_perf_enable(void) } val =0 ;if (event->hw.sample_period) { - left =atomic64_read (&event->hw.period_left); + left =local64_read (&event->hw.period_left);if (left <0x80000000L ) val =0x80000000L - left; } -atomic64_set (&event->hw.prev_count, val); +local64_set (&event->hw.prev_count, val); event->hw.idx = idx;write_pmc (idx, val);perf_event_update_userpage (event);
@@ -842,8 +842,8 @@ static void power_pmu_unthrottle(structif (left <0x80000000L ) val =0x80000000L - left;write_pmc (event->hw.idx, val); -atomic64_set (&event->hw.prev_count, val); -atomic64_set (&event->hw.period_left, left); +local64_set (&event->hw.prev_count, val); +local64_set (&event->hw.period_left, left);perf_event_update_userpage (event);perf_enable ();local_irq_restore (flags);
@@ -1108,7 +1108,7 @@ const struct pmu *hw_perf_event_init(str event->hw.config = events[n]; event->hw.event_base = cflags[n]; event->hw.last_period = event->hw.sample_period; -atomic64_set (&event->hw.period_left, event->hw.last_period); +local64_set (&event->hw.period_left, event->hw.last_period);/* * See if we need to reserve the PMU.
@@ -1146,16 +1146,16 @@ static void record_and_restart(struct peint record =0 ;/* we don't have to worry about interrupts here */ - prev =atomic64_read (&event->hw.prev_count); + prev =local64_read (&event->hw.prev_count); delta = (val - prev) &0xfffffffful ; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count);/* * See if the total period for this event has expired, * and update for the next period. */ val =0 ; - left =atomic64_read (&event->hw.period_left) - delta; + left =local64_read (&event->hw.period_left) - delta;if (period) {if (left <=0 ) { left += period;
@@ -1193,8 +1193,8 @@ static void record_and_restart(struct pe }write_pmc (event->hw.idx, val); -atomic64_set (&event->hw.prev_count, val); -atomic64_set (&event->hw.period_left, left); +local64_set (&event->hw.prev_count, val); +local64_set (&event->hw.period_left, left);perf_event_update_userpage (event); }
arch/sh/kernel/perf_event.c +3 -3
@@ -185,10 +185,10 @@ static void sh_perf_event_update(struct* this is the simplest approach for maintaining consistency. */ again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count = sh_pmu->read (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again;
@@ -203,7 +203,7 @@ again: delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); +local64_add (delta, &event->count); }static void sh_pmu_disable (struct perf_event *event)
arch/sparc/kernel/perf_event.c +9 -9
@@ -571,18 +571,18 @@ static u64 sparc_perf_event_update(strucs64 delta; again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count); new_raw_count =read_pmc (idx); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again; delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -590,27 +590,27 @@ again:static int sparc_perf_event_set_period (struct perf_event *event,struct hw_perf_event *hwc,int idx) { -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 ;if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (left >MAX_PERIOD ) left =MAX_PERIOD ; -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left);write_pmc (idx, (u64)(-left) &0xffffffff );
@@ -1086,7 +1086,7 @@ static int __hw_perf_event_init(struct pif (!hwc->sample_period) { hwc->sample_period =MAX_PERIOD ; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); }return 0 ;
arch/x86/kernel/cpu/perf_event.c +9 -9
@@ -294,10 +294,10 @@ x86_perf_event_update(struct perf_event* count to the generic event atomically: */ again: - prev_raw_count =atomic64_read (&hwc->prev_count); + prev_raw_count =local64_read (&hwc->prev_count);rdmsrl (hwc->event_base + idx, new_raw_count); -if (atomic64_cmpxchg (&hwc->prev_count, prev_raw_count, +if (local64_cmpxchg (&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again;
@@ -312,8 +312,8 @@ again: delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; -atomic64_add (delta, &event->count); -atomic64_sub (delta, &hwc->period_left); +local64_add (delta, &event->count); +local64_sub (delta, &hwc->period_left);return new_raw_count; }
@@ -437,7 +437,7 @@ static int x86_setup_perfctr(struct perfif (!hwc->sample_period) { hwc->sample_period = x86_pmu.max_period; hwc->last_period = hwc->sample_period; -atomic64_set (&hwc->period_left, hwc->sample_period); +local64_set (&hwc->period_left, hwc->sample_period); }else {/* * If we have a PMU initialized but no APIC
@@ -884,7 +884,7 @@ static int x86_perf_event_set_period(struct perf_event *event) {struct hw_perf_event *hwc = &event->hw; -s64 left =atomic64_read (&hwc->period_left); +s64 left =local64_read (&hwc->period_left);s64 period = hwc->sample_period;int ret =0 , idx = hwc->idx;
@@ -896,14 +896,14 @@ x86_perf_event_set_period(struct perf_ev*/ if (unlikely (left <= -period)) { left = period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }if (unlikely (left <=0 )) { left += period; -atomic64_set (&hwc->period_left, left); +local64_set (&hwc->period_left, left); hwc->last_period = period; ret =1 ; }
@@ -922,7 +922,7 @@ x86_perf_event_set_period(struct perf_ev* The hw event starts counting from this event offset, * mark it to be able to extra future deltas: */ -atomic64_set (&hwc->prev_count, (u64)-left); +local64_set (&hwc->prev_count, (u64)-left);wrmsrl (hwc->event_base + idx, (u64)(-left) & x86_pmu.cntval_mask);
include/linux/perf_event.h +4 -3
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {#include <linux/cpu.h> #include <asm/atomic.h> #include <asm/local.h> +#include <asm/local64.h> #define PERF_MAX_STACK_DEPTH 255
@@ -535,10 +536,10 @@ struct hw_perf_event {struct arch_hw_breakpoint info;#endif }; -atomic64_t prev_count; +local64_t prev_count;u64 sample_period;u64 last_period; -atomic64_t period_left; +local64_t period_left;u64 interrupts;u64 freq_time_stamp;
@@ -647,7 +648,7 @@ struct perf_event {const struct pmu *pmu;enum perf_event_active_state state; -atomic64_t count; +local64_t count;atomic64_t child_count;/*
kernel/perf_event.c +21 -21
141 lines not on this page · raw message
Quoted 10 lines · Peter Zijlstra
These patches introduce
local64_t.Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use
local64_tinstead ofatomic64_tand use instructions without buslock semantics.The
local64_timplementation useslocal_tfor 64 bits, sincelocal_tis of type long, for 32 bit it falls back toatomic64_t. Architectures can provide their own implementation as usual.
It seems nobody disagrees with it. Can we give it a try?
Thanks.
Peter Zijlstra
maintainerQuoted 8 lines · Frederic Weisbecker
Quoted 7 lines · Peter Zijlstra
These patches introduce
local64_t.Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use
local64_tinstead ofatomic64_tand use instructions without buslock semantics.
Quoted 6 lines
Quoted 3 lines
The
local64_timplementation useslocal_tfor 64 bits, sincelocal_tis of type long, for 32 bit it falls back toatomic64_t. Architectures can provide their own implementation as usual.It seems nobody disagrees with it. Can we give it a try?
I'll push it to mingo around -rc2 or so, to let the dust settle from the current merge.