Skip to contentSkip to conversation

convert perf to local64_t

PATCH In review Peter Zijlstra 5 participants 9 messages May 21, 2010 – May 26, 2010

Subsystems (15)
Arm (32-bit) x86 PowerPC MIPS +11 more subsystems
lore
Share & tools
lore
#0 lore raw

cover convert perf to local64_t

These patches introduce local64_t.

Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use local64_t instead of atomic64_t and use instructions without buslock semantics.

The local64_t implementation uses local_t for 64 bits, since local_t is of type long, for 32 bit it falls back to atomic64_t. Architectures can provide their own implementation as usual.

#1 lore raw

PATCH 1/4 arch: local64_t

Implements local64_t.

On 64bit, local_t is of size long, and thus we make local64_t an alias. On 32bit, we fall back to atomic64_t.

LKML-Reference: <new-submission>

---
 arch/alpha/include/asm/local64.h      |    1 
 arch/arm/include/asm/local64.h        |    1 
 arch/avr32/include/asm/local64.h      |    1 
 arch/blackfin/include/asm/local64.h   |    1 
 arch/cris/include/asm/local64.h       |    1 
 arch/frv/include/asm/local64.h        |    1 
 arch/frv/kernel/local64.h             |    1 
 arch/h8300/include/asm/local64.h      |    1 
 arch/ia64/include/asm/local64.h       |    1 
 arch/m32r/include/asm/local64.h       |    1 
 arch/m68k/include/asm/local64.h       |    1 
 arch/microblaze/include/asm/local64.h |    1 
 arch/mips/include/asm/local64.h       |    1 
 arch/mn10300/include/asm/local64.h    |    1 
 arch/parisc/include/asm/local64.h     |    1 
 arch/powerpc/include/asm/local64.h    |    1 
 arch/s390/include/asm/local64.h       |    1 
 arch/score/include/asm/local64.h      |    1 
 arch/sh/include/asm/local64.h         |    1 
 arch/sparc/include/asm/local64.h      |    1 
 arch/x86/include/asm/local64.h        |    1 
 arch/xtensa/include/asm/local64.h     |    1 
 include/asm-generic/local64.h         |   96 ++++++++++++++++++++++++++++++++++
 23 files changed, 118 insertions(+)
arch/alpha/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/alpha/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/arm/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/arm/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/avr32/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/avr32/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/blackfin/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/blackfin/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/cris/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/cris/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/frv/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/frv/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/frv/kernel/local64.h new file +1 -0
Index: linux-2.6/arch/frv/kernel/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/h8300/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/h8300/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/ia64/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/ia64/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/m32r/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/m32r/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/m68k/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/m68k/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/microblaze/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/microblaze/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/mips/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/mips/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/mn10300/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/mn10300/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/parisc/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/parisc/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/powerpc/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/powerpc/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/s390/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/s390/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/score/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/score/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/sh/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/sh/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/sparc/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/sparc/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/x86/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/x86/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
arch/xtensa/include/asm/local64.h new file +1 -0
Index: linux-2.6/arch/xtensa/include/asm/local64.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/local64.h>
include/asm-generic/local64.h new file +96 -0

96 lines not on this page · raw message

#3 lore raw

Re: [PATCH 1/4] arch: local64_t

In reply to Peter Zijlstra
Quoted 4 lines · Peter Zijlstra

+ * This is the default implementation, which uses atomic64_t. Which is
+ * rather pointless. The whole point behind local64_t is that some processors
+ * can perform atomic adds and subtracts in a manner which is atomic wrt IRQs
+ * running on this CPU. local64_t allows exploitation of such capabilities.

Interesting... What FRV does in atomic64-ops.S should probably be rebranded local64_t, and atomic64_t ops be based on that in non-SMP mode.

What I did on FRV was to emulate LL/ST instructions using some of the excessive numbers of conditional bits to do so - but it only works on UP systems.

David

#4 lore raw

PATCH 2/4 perf: Add perf_event_count()

In reply to Peter Zijlstra

Create a helper function for those sites that want to read the event count.

LKML-Reference: <new-submission>

---
 kernel/perf_event.c |   17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)
kernel/perf_event.c +11 -6
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -1701,6 +1701,11 @@ static void __perf_event_read(void *info
 	event->pmu->read(event);
 }
 
+static inline u64 perf_event_count(struct perf_event *event)
+{
+	return atomic64_read(&event->count);
+}
+
 static u64 perf_event_read(struct perf_event *event)
 {
 	/*
@@ -1720,7 +1725,7 @@ static u64 perf_event_read(struct perf_e
 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
 	}
 
-	return atomic64_read(&event->count);
+	return perf_event_count(event);
 }
 
 /*
@@ -2280,7 +2285,7 @@ void perf_event_update_userpage(struct p
 	++userpg->lock;
 	barrier();
 	userpg->index = perf_event_index(event);
-	userpg->offset = atomic64_read(&event->count);
+	userpg->offset = perf_event_count(event);
 	if (event->state == PERF_EVENT_STATE_ACTIVE)
 		userpg->offset -= atomic64_read(&event->hw.prev_count);
 
@@ -3125,7 +3130,7 @@ static void perf_output_read_one(struct 
 	u64 values[4];
 	int n = 0;
 
-	values[n++] = atomic64_read(&event->count);
+	values[n++] = perf_event_count(event);
 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
 		values[n++] = event->total_time_enabled +
 			atomic64_read(&event->child_total_time_enabled);
@@ -3162,7 +3167,7 @@ static void perf_output_read_group(struc
 	if (leader != event)
 		leader->pmu->read(leader);
 
-	values[n++] = atomic64_read(&leader->count);
+	values[n++] = perf_event_count(leader);
 	if (read_format & PERF_FORMAT_ID)
 		values[n++] = primary_event_id(leader);
 
@@ -3174,7 +3179,7 @@ static void perf_output_read_group(struc
 		if (sub != event)
 			sub->pmu->read(sub);
 
-		values[n++] = atomic64_read(&sub->count);
+		values[n++] = perf_event_count(sub);
 		if (read_format & PERF_FORMAT_ID)
 			values[n++] = primary_event_id(sub);
 
@@ -5272,7 +5277,7 @@ static void sync_child_event(struct perf
 	if (child_event->attr.inherit_stat)
 		perf_event_read_event(child_event, child);
 
-	child_val = atomic64_read(&child_event->count);
+	child_val = perf_event_count(child_event);
 
 	/*
 	 * Add back the child's count to the parent's count:
#5 lore raw

PATCH 3/4 perf: Add child_count

In reply to Peter Zijlstra

Only child counters adding back their values into the parent counter are responsible for cross-cpu updates to event->count.

So if we pull that out into a new child_count variable, we get an event->count that is only modified locally.

LKML-Reference: <new-submission>

---
 include/linux/perf_event.h |    1 +
 kernel/perf_event.c        |    4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
include/linux/perf_event.h +1 -0
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -648,6 +648,7 @@ struct perf_event {
 
 	enum perf_event_active_state	state;
 	atomic64_t			count;
+	atomic64_t			child_count;
 
 	/*
 	 * These are the total time in nanoseconds that the event
kernel/perf_event.c +2 -2
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -1703,7 +1703,7 @@ static void __perf_event_read(void *info
 
 static inline u64 perf_event_count(struct perf_event *event)
 {
-	return atomic64_read(&event->count);
+	return atomic64_read(&event->count) + atomic64_read(&event->child_count);
 }
 
 static u64 perf_event_read(struct perf_event *event)
@@ -5282,7 +5282,7 @@ static void sync_child_event(struct perf
 	/*
 	 * Add back the child's count to the parent's count:
 	 */
-	atomic64_add(child_val, &parent_event->count);
+	atomic64_add(child_val, &parent_event->child_count);
 	atomic64_add(child_event->total_time_enabled,
 		     &parent_event->child_total_time_enabled);
 	atomic64_add(child_event->total_time_running,
#6 lore raw

PATCH 4/4 perf: Convert perf_event to local_t

In reply to Peter Zijlstra

Since now all modification to event->count (and ->prev_count and ->period_left) are local to a cpu, change then to local64_t so we avoid the LOCK'ed ops.

LKML-Reference: <new-submission>

---
 arch/arm/kernel/perf_event.c     |   18 ++++++++--------
 arch/powerpc/kernel/perf_event.c |   34 +++++++++++++++----------------
 arch/sh/kernel/perf_event.c      |    6 ++---
 arch/sparc/kernel/perf_event.c   |   18 ++++++++--------
 arch/x86/kernel/cpu/perf_event.c |   18 ++++++++--------
 include/linux/perf_event.h       |    7 +++---
 kernel/perf_event.c              |   42 +++++++++++++++++++--------------------
 7 files changed, 72 insertions(+), 71 deletions(-)
arch/arm/kernel/perf_event.c +9 -9
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
@@ -128,20 +128,20 @@ armpmu_event_set_period(struct perf_even
 			struct hw_perf_event *hwc,
 			int idx)
 {
-	s64 left = atomic64_read(&hwc->period_left);
+	s64 left = local64_read(&hwc->period_left);
 	s64 period = hwc->sample_period;
 	int ret = 0;
 
 	if (unlikely(left <= -period)) {
 		left = period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 
 	if (unlikely(left <= 0)) {
 		left += period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
@@ -149,7 +149,7 @@ armpmu_event_set_period(struct perf_even
 	if (left > (s64)armpmu->max_period)
 		left = armpmu->max_period;
 
-	atomic64_set(&hwc->prev_count, (u64)-left);
+	local64_set(&hwc->prev_count, (u64)-left);
 
 	armpmu->write_counter(idx, (u64)(-left) & 0xffffffff);
 
@@ -168,18 +168,18 @@ armpmu_event_update(struct perf_event *e
 	s64 delta;
 
 again:
-	prev_raw_count = atomic64_read(&hwc->prev_count);
+	prev_raw_count = local64_read(&hwc->prev_count);
 	new_raw_count = armpmu->read_counter(idx);
 
-	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 			     new_raw_count) != prev_raw_count)
 		goto again;
 
 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 	delta >>= shift;
 
-	atomic64_add(delta, &event->count);
-	atomic64_sub(delta, &hwc->period_left);
+	local64_add(delta, &event->count);
+	local64_sub(delta, &hwc->period_left);
 
 	return new_raw_count;
 }
@@ -433,7 +433,7 @@ __hw_perf_event_init(struct perf_event *
 	if (!hwc->sample_period) {
 		hwc->sample_period  = armpmu->max_period;
 		hwc->last_period    = hwc->sample_period;
-		atomic64_set(&hwc->period_left, hwc->sample_period);
+		local64_set(&hwc->period_left, hwc->sample_period);
 	}
 
 	err = 0;
arch/powerpc/kernel/perf_event.c +17 -17
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
@@ -410,15 +410,15 @@ static void power_pmu_read(struct perf_e
 	 * Therefore we treat them like NMIs.
 	 */
 	do {
-		prev = atomic64_read(&event->hw.prev_count);
+		prev = local64_read(&event->hw.prev_count);
 		barrier();
 		val = read_pmc(event->hw.idx);
-	} while (atomic64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
+	} while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
 
 	/* The counters are only 32 bits wide */
 	delta = (val - prev) & 0xfffffffful;
-	atomic64_add(delta, &event->count);
-	atomic64_sub(delta, &event->hw.period_left);
+	local64_add(delta, &event->count);
+	local64_sub(delta, &event->hw.period_left);
 }
 
 /*
@@ -444,10 +444,10 @@ static void freeze_limited_counters(stru
 		if (!event->hw.idx)
 			continue;
 		val = (event->hw.idx == 5) ? pmc5 : pmc6;
-		prev = atomic64_read(&event->hw.prev_count);
+		prev = local64_read(&event->hw.prev_count);
 		event->hw.idx = 0;
 		delta = (val - prev) & 0xfffffffful;
-		atomic64_add(delta, &event->count);
+		local64_add(delta, &event->count);
 	}
 }
 
@@ -462,7 +462,7 @@ static void thaw_limited_counters(struct
 		event = cpuhw->limited_counter[i];
 		event->hw.idx = cpuhw->limited_hwidx[i];
 		val = (event->hw.idx == 5) ? pmc5 : pmc6;
-		atomic64_set(&event->hw.prev_count, val);
+		local64_set(&event->hw.prev_count, val);
 		perf_event_update_userpage(event);
 	}
 }
@@ -666,11 +666,11 @@ void hw_perf_enable(void)
 		}
 		val = 0;
 		if (event->hw.sample_period) {
-			left = atomic64_read(&event->hw.period_left);
+			left = local64_read(&event->hw.period_left);
 			if (left < 0x80000000L)
 				val = 0x80000000L - left;
 		}
-		atomic64_set(&event->hw.prev_count, val);
+		local64_set(&event->hw.prev_count, val);
 		event->hw.idx = idx;
 		write_pmc(idx, val);
 		perf_event_update_userpage(event);
@@ -842,8 +842,8 @@ static void power_pmu_unthrottle(struct 
 	if (left < 0x80000000L)
 		val = 0x80000000L - left;
 	write_pmc(event->hw.idx, val);
-	atomic64_set(&event->hw.prev_count, val);
-	atomic64_set(&event->hw.period_left, left);
+	local64_set(&event->hw.prev_count, val);
+	local64_set(&event->hw.period_left, left);
 	perf_event_update_userpage(event);
 	perf_enable();
 	local_irq_restore(flags);
@@ -1108,7 +1108,7 @@ const struct pmu *hw_perf_event_init(str
 	event->hw.config = events[n];
 	event->hw.event_base = cflags[n];
 	event->hw.last_period = event->hw.sample_period;
-	atomic64_set(&event->hw.period_left, event->hw.last_period);
+	local64_set(&event->hw.period_left, event->hw.last_period);
 
 	/*
 	 * See if we need to reserve the PMU.
@@ -1146,16 +1146,16 @@ static void record_and_restart(struct pe
 	int record = 0;
 
 	/* we don't have to worry about interrupts here */
-	prev = atomic64_read(&event->hw.prev_count);
+	prev = local64_read(&event->hw.prev_count);
 	delta = (val - prev) & 0xfffffffful;
-	atomic64_add(delta, &event->count);
+	local64_add(delta, &event->count);
 
 	/*
 	 * See if the total period for this event has expired,
 	 * and update for the next period.
 	 */
 	val = 0;
-	left = atomic64_read(&event->hw.period_left) - delta;
+	left = local64_read(&event->hw.period_left) - delta;
 	if (period) {
 		if (left <= 0) {
 			left += period;
@@ -1193,8 +1193,8 @@ static void record_and_restart(struct pe
 	}
 
 	write_pmc(event->hw.idx, val);
-	atomic64_set(&event->hw.prev_count, val);
-	atomic64_set(&event->hw.period_left, left);
+	local64_set(&event->hw.prev_count, val);
+	local64_set(&event->hw.period_left, left);
 	perf_event_update_userpage(event);
 }
 
arch/sh/kernel/perf_event.c +3 -3
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
@@ -185,10 +185,10 @@ static void sh_perf_event_update(struct 
 	 * this is the simplest approach for maintaining consistency.
 	 */
 again:
-	prev_raw_count = atomic64_read(&hwc->prev_count);
+	prev_raw_count = local64_read(&hwc->prev_count);
 	new_raw_count = sh_pmu->read(idx);
 
-	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 			     new_raw_count) != prev_raw_count)
 		goto again;
 
@@ -203,7 +203,7 @@ again:
 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 	delta >>= shift;
 
-	atomic64_add(delta, &event->count);
+	local64_add(delta, &event->count);
 }
 
 static void sh_pmu_disable(struct perf_event *event)
arch/sparc/kernel/perf_event.c +9 -9
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
@@ -571,18 +571,18 @@ static u64 sparc_perf_event_update(struc
 	s64 delta;
 
 again:
-	prev_raw_count = atomic64_read(&hwc->prev_count);
+	prev_raw_count = local64_read(&hwc->prev_count);
 	new_raw_count = read_pmc(idx);
 
-	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 			     new_raw_count) != prev_raw_count)
 		goto again;
 
 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 	delta >>= shift;
 
-	atomic64_add(delta, &event->count);
-	atomic64_sub(delta, &hwc->period_left);
+	local64_add(delta, &event->count);
+	local64_sub(delta, &hwc->period_left);
 
 	return new_raw_count;
 }
@@ -590,27 +590,27 @@ again:
 static int sparc_perf_event_set_period(struct perf_event *event,
 				       struct hw_perf_event *hwc, int idx)
 {
-	s64 left = atomic64_read(&hwc->period_left);
+	s64 left = local64_read(&hwc->period_left);
 	s64 period = hwc->sample_period;
 	int ret = 0;
 
 	if (unlikely(left <= -period)) {
 		left = period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 
 	if (unlikely(left <= 0)) {
 		left += period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 	if (left > MAX_PERIOD)
 		left = MAX_PERIOD;
 
-	atomic64_set(&hwc->prev_count, (u64)-left);
+	local64_set(&hwc->prev_count, (u64)-left);
 
 	write_pmc(idx, (u64)(-left) & 0xffffffff);
 
@@ -1086,7 +1086,7 @@ static int __hw_perf_event_init(struct p
 	if (!hwc->sample_period) {
 		hwc->sample_period = MAX_PERIOD;
 		hwc->last_period = hwc->sample_period;
-		atomic64_set(&hwc->period_left, hwc->sample_period);
+		local64_set(&hwc->period_left, hwc->sample_period);
 	}
 
 	return 0;
arch/x86/kernel/cpu/perf_event.c +9 -9
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
@@ -294,10 +294,10 @@ x86_perf_event_update(struct perf_event 
 	 * count to the generic event atomically:
 	 */
 again:
-	prev_raw_count = atomic64_read(&hwc->prev_count);
+	prev_raw_count = local64_read(&hwc->prev_count);
 	rdmsrl(hwc->event_base + idx, new_raw_count);
 
-	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 					new_raw_count) != prev_raw_count)
 		goto again;
 
@@ -312,8 +312,8 @@ again:
 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
 	delta >>= shift;
 
-	atomic64_add(delta, &event->count);
-	atomic64_sub(delta, &hwc->period_left);
+	local64_add(delta, &event->count);
+	local64_sub(delta, &hwc->period_left);
 
 	return new_raw_count;
 }
@@ -437,7 +437,7 @@ static int x86_setup_perfctr(struct perf
 	if (!hwc->sample_period) {
 		hwc->sample_period = x86_pmu.max_period;
 		hwc->last_period = hwc->sample_period;
-		atomic64_set(&hwc->period_left, hwc->sample_period);
+		local64_set(&hwc->period_left, hwc->sample_period);
 	} else {
 		/*
 		 * If we have a PMU initialized but no APIC
@@ -884,7 +884,7 @@ static int
 x86_perf_event_set_period(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
-	s64 left = atomic64_read(&hwc->period_left);
+	s64 left = local64_read(&hwc->period_left);
 	s64 period = hwc->sample_period;
 	int ret = 0, idx = hwc->idx;
 
@@ -896,14 +896,14 @@ x86_perf_event_set_period(struct perf_ev
 	 */
 	if (unlikely(left <= -period)) {
 		left = period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 
 	if (unlikely(left <= 0)) {
 		left += period;
-		atomic64_set(&hwc->period_left, left);
+		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
@@ -922,7 +922,7 @@ x86_perf_event_set_period(struct perf_ev
 	 * The hw event starts counting from this event offset,
 	 * mark it to be able to extra future deltas:
 	 */
-	atomic64_set(&hwc->prev_count, (u64)-left);
+	local64_set(&hwc->prev_count, (u64)-left);
 
 	wrmsrl(hwc->event_base + idx,
 			(u64)(-left) & x86_pmu.cntval_mask);
include/linux/perf_event.h +4 -3
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {
 #include <linux/cpu.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
+#include <asm/local64.h>
 
 #define PERF_MAX_STACK_DEPTH		255
 
@@ -535,10 +536,10 @@ struct hw_perf_event {
 		struct arch_hw_breakpoint	info;
 #endif
 	};
-	atomic64_t			prev_count;
+	local64_t			prev_count;
 	u64				sample_period;
 	u64				last_period;
-	atomic64_t			period_left;
+	local64_t			period_left;
 	u64				interrupts;
 
 	u64				freq_time_stamp;
@@ -647,7 +648,7 @@ struct perf_event {
 	const struct pmu		*pmu;
 
 	enum perf_event_active_state	state;
-	atomic64_t			count;
+	local64_t			count;
 	atomic64_t			child_count;
 
 	/*
kernel/perf_event.c +21 -21

141 lines not on this page · raw message

#7 lore raw
In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra

These patches introduce local64_t.

Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use local64_t instead of atomic64_t and use instructions without buslock semantics.

The local64_t implementation uses local_t for 64 bits, since local_t is of type long, for 32 bit it falls back to atomic64_t. Architectures can provide their own implementation as usual.

It seems nobody disagrees with it. Can we give it a try?

Thanks.

maintainer
#8 lore raw
Quoted 8 lines · Frederic Weisbecker
Quoted 7 lines · Peter Zijlstra

These patches introduce local64_t.

Since perf_event:count is only modified cross-cpu when child-counters feed back their changes on exit, and we can use a secondary variable for that, we can convert perf to use local64_t instead of atomic64_t and use instructions without buslock semantics.

Quoted 6 lines
Quoted 3 lines

The local64_t implementation uses local_t for 64 bits, since local_t is of type long, for 32 bit it falls back to atomic64_t. Architectures can provide their own implementation as usual.

It seems nobody disagrees with it. Can we give it a try?

I'll push it to mingo around -rc2 or so, to let the dust settle from the current merge.