Skip to contentSkip to conversation

perf_events: fix bug in hw_perf_enable()

PATCH In review Stephane Eranian 2 participants 13 messages Feb 1, 2010 – Feb 4, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 perf_events: fix bug in hw_perf_enable()

We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().

The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.

This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.

--

 arch/x86/kernel/cpu/perf_event.c |   38 ++++++++++++++++++++++++++++++--------
 include/linux/perf_event.h       |    2 ++
 2 files changed, 32 insertions(+), 8 deletions(-)
arch/x86/kernel/cpu/perf_event.c +30 -8
index 1846ead..c87bf1e 100644
@@ -90,6 +90,7 @@ struct cpu_hw_events {
 	int			n_events;
 	int			n_added;
 	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
+	u64			tags[X86_PMC_IDX_MAX];
 	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
 };
 
@@ -1128,6 +1129,8 @@ static int __hw_perf_event_init(struct perf_event *event)
 	hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 
 	hwc->idx = -1;
+	hwc->last_cpu = -1;
+	hwc->last_tag = ~0ULL;
 
 	/*
 	 * Count user and OS events unless requested not to.
@@ -1443,11 +1446,14 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,
 	return n;
 }
 
-
 static inline void x86_assign_hw_event(struct perf_event *event,
-				struct hw_perf_event *hwc, int idx)
+				struct cpu_hw_events *cpuc, int i)
 {
-	hwc->idx = idx;
+	struct hw_perf_event *hwc = &event->hw;
+
+	hwc->idx = cpuc->assign[i];
+	hwc->last_cpu = smp_processor_id();
+	hwc->last_tag = ++cpuc->tags[i];
 
 	if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
 		hwc->config_base = 0;
@@ -1466,6 +1472,15 @@ static inline void x86_assign_hw_event(struct perf_event *event,
 	}
 }
 
+static inline int match_prev_assignment(struct hw_perf_event *hwc,
+					struct cpu_hw_events *cpuc,
+					int i)
+{
+	return hwc->idx == cpuc->assign[i]
+	 && hwc->last_cpu == smp_processor_id()
+	 && hwc->last_tag == cpuc->tags[i];
+}
+
 static void __x86_pmu_disable(struct perf_event *event, struct cpu_hw_events *cpuc);
 
 void hw_perf_enable(void)
@@ -1494,8 +1509,15 @@ void hw_perf_enable(void)
 			event = cpuc->event_list[i];
 			hwc = &event->hw;
 
-			if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
-				continue;
+			/*
+			 * we can avoid reprogramming counter if:
+			 * - assigned same counter as last time
+			 * - running on same CPU as last time
+			 * - no other event has used the counter since
+			 */
+			if (hwc->idx == -1
+			    || match_prev_assignment(hwc, cpuc, i))
+                                continue;
 
 			__x86_pmu_disable(event, cpuc);
 
@@ -1508,12 +1530,12 @@ void hw_perf_enable(void)
 			hwc = &event->hw;
 
 			if (hwc->idx == -1) {
-				x86_assign_hw_event(event, hwc, cpuc->assign[i]);
-				x86_perf_event_set_period(event, hwc, hwc->idx);
+				x86_assign_hw_event(event, cpuc, i);
+ 				x86_perf_event_set_period(event, hwc, hwc->idx);
 			}
 			/*
 			 * need to mark as active because x86_pmu_disable()
-			 * clear active_mask and eventsp[] yet it preserves
+			 * clear active_mask and events[] yet it preserves
 			 * idx
 			 */
 			set_bit(hwc->idx, cpuc->active_mask);
include/linux/perf_event.h +2 -0
index 556b0f4..071a7db 100644
@@ -478,9 +478,11 @@ struct hw_perf_event {
 	union {
 		struct { /* hardware */
 			u64		config;
+			u64		last_tag;
 			unsigned long	config_base;
 			unsigned long	event_base;
 			int		idx;
+			int		last_cpu;
 		};
 		struct { /* software */
 			s64		remaining;
maintainer
#1 lore raw
Quoted 8 lines · Stephane Eranian

We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().

The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.

Quoted 5 lines

This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.

Thanks, got it.

btw, I've also added the below, from what I can make from the docs fixed counter 2 is identical to arch perf event 0x013c, as per table A-1 and A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where 0x013c is called CPU_CLK_UNHALTED.BUS.

---
Subject: perf_events, x86: Fixup fixed counter constraints
From: Peter Zijlstra <[email protected]>
Date: Mon Feb 01 15:36:30 CET 2010

Patch 1da53e0230 ("perf_events, x86: Improve x86 event scheduling") lost us one of the fixed purpose counters and then ed8777fc13 ("perf_events, x86: Fix event constraint masks") broke it even further.

Widen the fixed event mask to event+umask and specify the full config for each of the 3 fixed purpose counters. Then let the init code fill out the placement for the GP regs based on the cpuid info.

LKML-Reference: <new-submission>

---
 arch/x86/include/asm/perf_event.h |    2 +-
 arch/x86/kernel/cpu/perf_event.c  |   38 ++++++++++++++++++++++++++++++--------
 2 files changed, 31 insertions(+), 9 deletions(-)
arch/x86/include/asm/perf_event.h +1 -1
Index: linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================
@@ -50,7 +50,7 @@
 	 INTEL_ARCH_INV_MASK| \
 	 INTEL_ARCH_EDGE_MASK|\
 	 INTEL_ARCH_UNIT_MASK|\
-	 INTEL_ARCH_EVTSEL_MASK)
+	 INTEL_ARCH_EVENT_MASK)
 
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL		      0x3c
 #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK		(0x00 << 8)
arch/x86/kernel/cpu/perf_event.c +30 -8
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
@@ -243,8 +243,18 @@ static struct event_constraint intel_cor
 
 static struct event_constraint intel_core2_event_constraints[] =
 {
-	FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-	FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+	FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+	/*
+	 * FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34),  CPU_CLK_UNHALTED.REF
+	 *
+	 * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+	 * 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+	 * ratio between these counters.
+	 *
+	 * TODO: find/measure the fixed ratio and apply it so that we can
+	 * enable this fixed purpose counter in a transparent way.
+	 */
 	INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
 	INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
 	INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
@@ -259,8 +269,9 @@ static struct event_constraint intel_cor
 
 static struct event_constraint intel_nehalem_event_constraints[] =
 {
-	FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-	FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+	FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+	FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */
 	INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
 	INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
 	INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
@@ -274,8 +285,9 @@ static struct event_constraint intel_neh
 
 static struct event_constraint intel_westmere_event_constraints[] =
 {
-	FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-	FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+	FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+	FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */
 	INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
 	INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
 	INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
@@ -284,8 +296,9 @@ static struct event_constraint intel_wes
 
 static struct event_constraint intel_gen_event_constraints[] =
 {
-	FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-	FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+	FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+	FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */
 	EVENT_CONSTRAINT_END
 };
 
@@ -2602,6 +2615,7 @@ static void __init pmu_check_apic(void)
 
 void __init init_hw_perf_events(void)
 {
+	struct event_constraint *c;
 	int err;
 
 	pr_info("Performance Events: ");
@@ -2650,6 +2664,14 @@ void __init init_hw_perf_events(void)
 		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1,
 				   0, x86_pmu.num_events);
 
+	for_each_event_constraint(c, x86_pmu.event_constraints) {
+		if (c->cmask != INTEL_ARCH_FIXED_MASK)
+			continue;
+
+		c->idxmsk64[0] |= (1ULL << x86_pmu.num_events) - 1;
+		c->weight += x86_pmu.num_events;
+	}
+
 	pr_info("... version:                %d\n",     x86_pmu.version);
 	pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
 	pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
maintainer
#2 lore raw
Quoted 23 lines · Peter Zijlstra

+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c

arch/x86/kernel/cpu/perf_event.c +12 -2
@@ -243,8 +243,18 @@ static struct event_constraint intel_cor
 
 static struct event_constraint intel_core2_event_constraints[] =
 {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       /*
+        * FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34),  CPU_CLK_UNHALTED.REF
+        *
+        * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+        * 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+        * ratio between these counters.
+        *
+        * TODO: find/measure the fixed ratio and apply it so that we can
+        * enable this fixed purpose counter in a transparent way.
+        */
        INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
        INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
        INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ 
Quoted 1 line

From what I can measure on the available Core2 systems this ratio is

exactly 1, which would be consistent with the Nehalem and Westmere tables calling this event .REF

Stephane, have you ever observed this ratio to be anything other than 1?

If not, I think we can simply stick this counter back in and not worry about it.

#3 lore raw
Quoted 24 lines · Peter Zijlstra
Quoted 22 lines · Peter Zijlstra

+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c

(quoted diff) +0 -0
@@ -243,8 +243,18 @@ static struct event_constraint intel_cor
 

 static struct event_constraint intel_core2_event_constraints[] =
 {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       /*
+        * FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34),  CPU_CLK_UNHALTED.REF
+        *
+        * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+        * 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+        * ratio between these counters.
+        *
+        * TODO: find/measure the fixed ratio and apply it so that we can
+        * enable this fixed purpose counter in a transparent way.
+        */
        INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */ INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */

Quoted 6 lines
Quoted 1 line

From what I can measure on the available Core2 systems this ratio is

exactly 1, which would be consistent with the Nehalem and Westmere tables calling this event .REF

Stephane, have you ever observed this ratio to be anything other than 1?

Using perfmon on Core 2 on a 10s noploop:

pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus
noploop 10
noploop for 10 seconds
23869090125 UNHALTED_REFERENCE_CYCLES
23849336873 UNHALTED_CORE_CYCLES
2652122099 CPU_CLK_UNHALTED:BUS

Quoted 4 lines

If not, I think we can simply stick this counter back in and not worry about it.

Signature · 8 lines
Stephane Eranian  | EMEA Software Engineering
Google France | 38 avenue de l'Opéra | 75002 Paris
Tel : +33 (0) 1 42 68 53 00
This email may be confidential or privileged. If you received this
communication by mistake, please
don't forward it to anyone else, please erase all copies and
attachments, and please let me know that
it went to the wrong person. Thanks
maintainer
#4 lore raw
Quoted 8 lines · Stephane Eranian

Using perfmon on Core 2 on a 10s noploop:

pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus
noploop 10
noploop for 10 seconds
23869090125 UNHALTED_REFERENCE_CYCLES
23849336873 UNHALTED_CORE_CYCLES
2652122099 CPU_CLK_UNHALTED:BUS

Weird, I used:

while :; do :; done &
while :; do :; done &
while :; do :; done &
while :; do :; done &
perf stat -a -e r013c -e r013c sleep 4
killall bash

Which gives:

Performance counter stats for 'sleep 4':

244235699509090 raw 0x13c
244235695558036 raw 0x13c

    4.005485333  seconds time elapsed

And verified it used fixed counter 2 and general purpose counter 0 using sysrq-p.

[523417.108402] CPU#0:   gen-PMC0 ctrl:  000000000053013c
[523417.108403] CPU#0:   gen-PMC0 count: 000000ff80019948
[523417.108405] CPU#0:   gen-PMC0 left:  000000007fffffff
[523417.108407] CPU#0:   gen-PMC1 ctrl:  0000000000000000
[523417.108409] CPU#0:   gen-PMC1 count: 0000000000000000
[523417.108411] CPU#0:   gen-PMC1 left:  000000007fffb8a8
[523417.108412] CPU#0: fixed-PMC0 count: 0000000000000000
[523417.108414] CPU#0: fixed-PMC1 count: 0000000000000000
[523417.108416] CPU#0: fixed-PMC2 count: 0000010db1db2117

Using -linus, since that doesn't have any of the recent constraint patches in that would avoid us from using fixed-PMC2.

#5 lore raw
Quoted 20 lines · Peter Zijlstra
Quoted 8 lines · Stephane Eranian

Using perfmon on Core 2 on a 10s noploop:

pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus noploop 10
noploop for 10 seconds
23869090125 UNHALTED_REFERENCE_CYCLES
23849336873 UNHALTED_CORE_CYCLES
 2652122099 CPU_CLK_UNHALTED:BUS

Weird, I used:

 while :; do :; done &
 while :; do :; done &
 while :; do :; done &
 while :; do :; done &
 perf stat -a -e r013c -e r013c sleep 4
 killall bash

Quoted 7 lines

Which gives:

 Performance counter stats for 'sleep 4':

 244235699509090  raw 0x13c
 244235695558036  raw 0x13c

Strange. I am running mine on a Q6600.
What's yours?

Quoted 19 lines

   4.005485333  seconds time elapsed

And verified it used fixed counter 2 and general purpose counter 0 using sysrq-p.

[523417.108402] CPU#0:   gen-PMC0 ctrl:  000000000053013c
[523417.108403] CPU#0:   gen-PMC0 count: 000000ff80019948
[523417.108405] CPU#0:   gen-PMC0 left:  000000007fffffff
[523417.108407] CPU#0:   gen-PMC1 ctrl:  0000000000000000
[523417.108409] CPU#0:   gen-PMC1 count: 0000000000000000
[523417.108411] CPU#0:   gen-PMC1 left:  000000007fffb8a8
[523417.108412] CPU#0: fixed-PMC0 count: 0000000000000000
[523417.108414] CPU#0: fixed-PMC1 count: 0000000000000000
[523417.108416] CPU#0: fixed-PMC2 count: 0000010db1db2117

Using -linus, since that doesn't have any of the recent constraint patches in that would avoid us from using fixed-PMC2.

Signature · 8 lines
Stephane Eranian  | EMEA Software Engineering
Google France | 38 avenue de l'Opéra | 75002 Paris
Tel : +33 (0) 1 42 68 53 00
This email may be confidential or privileged. If you received this
communication by mistake, please
don't forward it to anyone else, please erase all copies and
attachments, and please let me know that
it went to the wrong person. Thanks
maintainer
#6 lore raw
Quoted 5 lines · Stephane Eranian
Quoted 3 lines

244235699509090 raw 0x13c
244235695558036 raw 0x13c

Strange. I am running mine on a Q6600.
What's yours?

These numbers came from a T9400, but my Q9450 does something similar.

#7 lore raw
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 3 lines

 244235699509090  raw 0x13c
 244235695558036  raw 0x13c

Strange. I am running mine on a Q6600.
What's yours?

These numbers came from a T9400, but my Q9450 does something similar.

Are you sure you are reading from the correct corresponding data registers?

maintainer
#8 lore raw
Quoted 11 lines · Stephane Eranian
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 3 lines

244235699509090 raw 0x13c
244235695558036 raw 0x13c

Strange. I am running mine on a Q6600.
What's yours?

These numbers came from a T9400, but my Q9450 does something similar.

Are you sure you are reading from the correct corresponding data registers?

It sure got me wondering, I'll go over that fixed counter code in arch/x86/kernel/cpu/perf_event.c in the morning.

#9 lore raw
In reply to Peter Zijlstra
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Stephane Eranian

      We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().

      The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.

      This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.

Quoted 9 lines
Quoted 1 line

Thanks, got it.

btw, I've also added the below, from what I can make from the docs fixed counter 2 is identical to arch perf event 0x013c, as per table A-1 and A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where 0x013c is called CPU_CLK_UNHALTED.BUS.

If you measure 0x013c in a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.

This goes back to an earlier discussion where I was asking about the meaning of the generic PMU events and in particular PERF_COUNT_HW_CPU_CYCLES. Which of the 3 distinct cycle events (unhalted_core_cycles, unhalted_reference_cycles, bus_cycles) does not correspond to?

Quoted 121 lines

---
Subject: perf_events, x86: Fixup fixed counter constraints
From: Peter Zijlstra <[email protected]>
Date: Mon Feb 01 15:36:30 CET 2010

Patch 1da53e0230 ("perf_events, x86: Improve x86 event scheduling") lost us one of the fixed purpose counters and then ed8777fc13 ("perf_events, x86: Fix event constraint masks") broke it even further.

Widen the fixed event mask to event+umask and specify the full config for each of the 3 fixed purpose counters. Then let the init code fill out the placement for the GP regs based on the cpuid info.

LKML-Reference: <new-submission>
---
 arch/x86/include/asm/perf_event.h |    2 +- arch/x86/kernel/cpu/perf_event.c  |   38 ++++++++++++++++++++++++++++++--------

 2 files changed, 31 insertions(+), 9 deletions(-)

Index: linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================

arch/x86/include/asm/perf_event.h +1 -1
@@ -50,7 +50,7 @@
         INTEL_ARCH_INV_MASK| \
         INTEL_ARCH_EDGE_MASK|\
         INTEL_ARCH_UNIT_MASK|\
-        INTEL_ARCH_EVTSEL_MASK)
+        INTEL_ARCH_EVENT_MASK)
 
  #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL                0x3c
  #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK                (0x00 << 8)

Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================

arch/x86/kernel/cpu/perf_event.c +0 -0
@@ -243,8 +243,18 @@ static struct event_constraint intel_cor
 

 static struct event_constraint intel_core2_event_constraints[] =
 {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       /*
+        * FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34),  CPU_CLK_UNHALTED.REF
+        *
+        * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+        * 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+        * ratio between these counters.
+        *
+        * TODO: find/measure the fixed ratio and apply it so that we can
+        * enable this fixed purpose counter in a transparent way.
+        */
       INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */ INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */

arch/x86/kernel/cpu/perf_event.c +0 -0
@@ -259,8 +269,9 @@ static struct event_constraint intel_cor
 

 static struct event_constraint intel_nehalem_event_constraints[] =
 {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */ INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */ INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */ INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */

arch/x86/kernel/cpu/perf_event.c +0 -0
@@ -274,8 +285,9 @@ static struct event_constraint intel_neh
 

 static struct event_constraint intel_westmere_event_constraints[] = {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */ INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */ INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */ INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */

arch/x86/kernel/cpu/perf_event.c +0 -0
@@ -284,8 +296,9 @@ static struct event_constraint intel_wes
 

 static struct event_constraint intel_gen_event_constraints[] =
 {
-       FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
-       FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
+       FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+       FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+       FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */ EVENT_CONSTRAINT_END
 };

arch/x86/kernel/cpu/perf_event.c +9 -0
@@ -2602,6 +2615,7 @@ static void __init pmu_check_apic(void)
 
  void __init init_hw_perf_events(void)
  {
+       struct event_constraint *c;
        int err;
 
        pr_info("Performance Events: ");
@@ -2650,6 +2664,14 @@ void __init init_hw_perf_events(void)
                __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1,
                                   0, x86_pmu.num_events);
 
+       for_each_event_constraint(c, x86_pmu.event_constraints) {
+               if (c->cmask != INTEL_ARCH_FIXED_MASK)
+                       continue;
+
+               c->idxmsk64[0] |= (1ULL << x86_pmu.num_events) - 1;
+               c->weight += x86_pmu.num_events;
+       }
+
        pr_info("... version:                %d\n",     x86_pmu.version);
        pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
        pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
Signature · 8 lines
Stephane Eranian  | EMEA Software Engineering
Google France | 38 avenue de l'Opéra | 75002 Paris
Tel : +33 (0) 1 42 68 53 00
This email may be confidential or privileged. If you received this
communication by mistake, please
don't forward it to anyone else, please erase all copies and
attachments, and please let me know that
it went to the wrong person. Thanks
maintainer
#10 lore raw
Quoted 4 lines · Stephane Eranian
Quoted 1 line

btw, I've also added the below, from what I can make from the docs

fixed

Quoted 1 line

counter 2 is identical to arch perf event 0x013c, as per table A-1

and

Quoted 8 lines
Quoted 3 lines

A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where 0x013c is called CPU_CLK_UNHALTED.BUS.

If you measure 0x013c in a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.

But for Nehalem and Westmere event 0x3c umask 0x01 is referred to as CPU_CLK_UNHALTED.REF_P (Tables A-2 and A-4),

Fixed Counter 2 is referred to as CPU_CLK_UNHALTED.REF (Table A-7).

For Core2 and Atom (Table A-8, A-9) it is called CPU_CLK_UNHALTED.BUS, for these entries there is talk about a fixed ratio.

#11 lore raw
Quoted 9 lines · Peter Zijlstra
Quoted 8 lines · Stephane Eranian
Quoted 1 line

btw, I've also added the below, from what I can make from the docs

fixed

Quoted 1 line

counter 2 is identical to arch perf event 0x013c, as per table A-1

and

Quoted 3 lines

A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where 0x013c is called CPU_CLK_UNHALTED.BUS.

Quoted 8 lines
Quoted 4 lines

If you measure 0x013c in a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.

But for Nehalem and Westmere event 0x3c umask 0x01 is referred to as CPU_CLK_UNHALTED.REF_P (Tables A-2 and A-4),

_P means programmable (counter).

on my NHM:

pfmon -v -eunhalted_core_cycles,unhalted_reference_cycles,cpu_clk_unhalted:ref_p
./noploop 10
[FIXED_CTRL(pmc16)=0xaa0 pmi0=1 en0=0x0 any0=0 pmi1=1 en1=0x2 any1=0 pmi2=1 en2=0x2 any2=0] UNHALTED_CORE_CYCLES UNHALTED_REFERENCE_CYCLES [PERFEVTSEL0(pmc0)=0x51013c event_sel=0x3c umask=0x1 os=0 usr=1 anythr=0 en=1 int=1 inv=0 edge=0 cnt_mask=0] CPU_CLK_UNHALTED noploop for 10 seconds

29292225749 UNHALTED_CORE_CYCLES
29292249954 UNHALTED_REFERENCE_CYCLES
1331465398 CPU_CLK_UNHALTED:REF_P

REF_P runs at 133MHz. Core at 2.93GHz.

Quoted 6 lines

Fixed Counter 2 is referred to as CPU_CLK_UNHALTED.REF (Table A-7).

For Core2 and Atom (Table A-8, A-9) it is called CPU_CLK_UNHALTED.BUS, for these entries there is talk about a fixed ratio.

Signature · 8 lines
Stephane Eranian  | EMEA Software Engineering
Google France | 38 avenue de l'Opéra | 75002 Paris
Tel : +33 (0) 1 42 68 53 00
This email may be confidential or privileged. If you received this
communication by mistake, please
don't forward it to anyone else, please erase all copies and
attachments, and please let me know that
it went to the wrong person. Thanks
#12 lore raw

[tip:perf/core] perf_events, x86: Fix bug in hw_perf_enable()

In reply to Stephane Eranian
Commit-ID:  447a194b393f32699607fd99617a40abd6a95114
Gitweb:     http://git.kernel.org/tip/447a194b393f32699607fd99617a40abd6a95114
Author:     Stephane Eranian <[email protected]>
AuthorDate: Mon, 1 Feb 2010 14:50:01 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Thu, 4 Feb 2010 09:59:50 +0100

perf_events, x86: Fix bug in hw_perf_enable()

We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().

The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.

This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.

Cc Mike Galbraith <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>

LKML-Reference: <[email protected]>

---
 arch/x86/kernel/cpu/perf_event.c |   34 ++++++++++++++++++++++++++++------
 include/linux/perf_event.h       |    2 ++
 2 files changed, 30 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_event.c +28 -6
index 96cfc1a..a920f17 100644
@@ -90,6 +90,7 @@ struct cpu_hw_events {
 	int			n_events;
 	int			n_added;
 	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
+	u64			tags[X86_PMC_IDX_MAX];
 	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
 };
 
@@ -1142,6 +1143,8 @@ static int __hw_perf_event_init(struct perf_event *event)
 	hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 
 	hwc->idx = -1;
+	hwc->last_cpu = -1;
+	hwc->last_tag = ~0ULL;
 
 	/*
 	 * Count user and OS events unless requested not to.
@@ -1457,11 +1460,14 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,
 	return n;
 }
 
-
 static inline void x86_assign_hw_event(struct perf_event *event,
-				struct hw_perf_event *hwc, int idx)
+				struct cpu_hw_events *cpuc, int i)
 {
-	hwc->idx = idx;
+	struct hw_perf_event *hwc = &event->hw;
+
+	hwc->idx = cpuc->assign[i];
+	hwc->last_cpu = smp_processor_id();
+	hwc->last_tag = ++cpuc->tags[i];
 
 	if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
 		hwc->config_base = 0;
@@ -1480,6 +1486,15 @@ static inline void x86_assign_hw_event(struct perf_event *event,
 	}
 }
 
+static inline int match_prev_assignment(struct hw_perf_event *hwc,
+					struct cpu_hw_events *cpuc,
+					int i)
+{
+	return hwc->idx == cpuc->assign[i] &&
+		hwc->last_cpu == smp_processor_id() &&
+		hwc->last_tag == cpuc->tags[i];
+}
+
 static void __x86_pmu_disable(struct perf_event *event, struct cpu_hw_events *cpuc);
 
 void hw_perf_enable(void)
@@ -1508,7 +1523,14 @@ void hw_perf_enable(void)
 			event = cpuc->event_list[i];
 			hwc = &event->hw;
 
-			if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
+			/*
+			 * we can avoid reprogramming counter if:
+			 * - assigned same counter as last time
+			 * - running on same CPU as last time
+			 * - no other event has used the counter since
+			 */
+			if (hwc->idx == -1 ||
+			    match_prev_assignment(hwc, cpuc, i))
 				continue;
 
 			__x86_pmu_disable(event, cpuc);
@@ -1522,12 +1544,12 @@ void hw_perf_enable(void)
 			hwc = &event->hw;
 
 			if (hwc->idx == -1) {
-				x86_assign_hw_event(event, hwc, cpuc->assign[i]);
+				x86_assign_hw_event(event, cpuc, i);
 				x86_perf_event_set_period(event, hwc, hwc->idx);
 			}
 			/*
 			 * need to mark as active because x86_pmu_disable()
-			 * clear active_mask and eventsp[] yet it preserves
+			 * clear active_mask and events[] yet it preserves
 			 * idx
 			 */
 			set_bit(hwc->idx, cpuc->active_mask);
include/linux/perf_event.h +2 -0
index 556b0f4..071a7db 100644
@@ -478,9 +478,11 @@ struct hw_perf_event {
 	union {
 		struct { /* hardware */
 			u64		config;
+			u64		last_tag;
 			unsigned long	config_base;
 			unsigned long	event_base;
 			int		idx;
+			int		last_cpu;
 		};
 		struct { /* software */
 			s64		remaining;