perf_events: fix bug in hw_perf_enable()
Subsystems (2)
PATCH 1/1 perf_events: fix bug in hw_perf_enable()
We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().
The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.
This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.
--
arch/x86/kernel/cpu/perf_event.c | 38 ++++++++++++++++++++++++++++++-------- include/linux/perf_event.h | 2 ++ 2 files changed, 32 insertions(+), 8 deletions(-)
arch/x86/kernel/cpu/perf_event.c +30 -8
@@ -90,6 +90,7 @@ struct cpu_hw_events {int n_events;int n_added;int assign[X86_PMC_IDX_MAX ];/* event to counter assignment */ +u64 tags[X86_PMC_IDX_MAX ];struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ };
@@ -1128,6 +1129,8 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->config =ARCH_PERFMON_EVENTSEL_INT ; hwc->idx =-1 ; + hwc->last_cpu =-1 ; + hwc->last_tag = ~0ULL ;/* * Count user and OS events unless requested not to.
@@ -1443,11 +1446,14 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,return n; } -static inline void x86_assign_hw_event (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +struct cpu_hw_events *cpuc,int i) { - hwc->idx = idx; +struct hw_perf_event *hwc = &event->hw; + + hwc->idx = cpuc->assign[i]; + hwc->last_cpu =smp_processor_id (); + hwc->last_tag = ++cpuc->tags[i];if (hwc->idx ==X86_PMC_IDX_FIXED_BTS ) { hwc->config_base =0 ;
@@ -1466,6 +1472,15 @@ static inline void x86_assign_hw_event(struct perf_event *event, } } +static inline int match_prev_assignment (struct hw_perf_event *hwc, +struct cpu_hw_events *cpuc, +int i) +{ +return hwc->idx == cpuc->assign[i] + && hwc->last_cpu ==smp_processor_id () + && hwc->last_tag == cpuc->tags[i]; +} +static void __x86_pmu_disable (struct perf_event *event,struct cpu_hw_events *cpuc);void hw_perf_enable (void )
@@ -1494,8 +1509,15 @@ void hw_perf_enable(void) event = cpuc->event_list[i]; hwc = &event->hw; -if (hwc->idx ==-1 || hwc->idx == cpuc->assign[i]) -continue ; +/* +* we can avoid reprogramming counter if: +* - assigned same counter as last time +* - running on same CPU as last time +* - no other event has used the counter since +*/ +if (hwc->idx ==-1 + ||match_prev_assignment (hwc, cpuc, i)) +continue ;__x86_pmu_disable (event, cpuc);
@@ -1508,12 +1530,12 @@ void hw_perf_enable(void) hwc = &event->hw;if (hwc->idx ==-1 ) { -x86_assign_hw_event (event, hwc, cpuc->assign[i]); -x86_perf_event_set_period (event, hwc, hwc->idx); +x86_assign_hw_event (event, cpuc, i); +x86_perf_event_set_period (event, hwc, hwc->idx); }/* * need to mark as active because x86_pmu_disable() -* clear active_mask and eventsp[] yet it preserves +* clear active_mask and events[] yet it preserves * idx */ set_bit (hwc->idx, cpuc->active_mask);
include/linux/perf_event.h +2 -0
@@ -478,9 +478,11 @@ struct hw_perf_event {union {struct {/* hardware */ u64 config; +u64 last_tag;unsigned long config_base;unsigned long event_base;int idx; +int last_cpu; };struct {/* software */ s64 remaining;
Peter Zijlstra
maintainerQuoted 8 lines · Stephane Eranian
We cannot assume that because
hwc->idx== assign[i], we can avoid reprogramming the counter inhw_perf_enable().The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.
Quoted 5 lines
This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.
Signed-off-by Stephane Eranian <[email protected]>
Thanks, got it.
btw, I've also added the below, from what I can make from the docs fixed counter 2 is identical to arch perf event 0x013c, as per table A-1 and A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where 0x013c is called CPU_CLK_UNHALTED.BUS.
---
Subject: perf_events, x86: Fixup fixed counter constraints
From: Peter Zijlstra <[email protected]>
Date: Mon Feb 01 15:36:30 CET 2010
Patch 1da53e0230 ("perf_events, x86: Improve x86 event scheduling") lost us one of the fixed purpose counters and then ed8777fc13 ("perf_events, x86: Fix event constraint masks") broke it even further.
Widen the fixed event mask to event+umask and specify the full config for each of the 3 fixed purpose counters. Then let the init code fill out the placement for the GP regs based on the cpuid info.
LKML-Reference: <new-submission>
--- arch/x86/include/asm/perf_event.h | 2 +- arch/x86/kernel/cpu/perf_event.c | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-)
arch/x86/include/asm/perf_event.h +1 -1
@@ -50,7 +50,7 @@ INTEL_ARCH_INV_MASK| \ INTEL_ARCH_EDGE_MASK|\ INTEL_ARCH_UNIT_MASK|\ - INTEL_ARCH_EVTSEL_MASK) + INTEL_ARCH_EVENT_MASK)#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)
arch/x86/kernel/cpu/perf_event.c +30 -8
@@ -243,8 +243,18 @@ static struct event_constraint intel_corstatic struct event_constraint intel_core2_event_constraints[] = { -FIXED_EVENT_CONSTRAINT (0xc0 , (0x3 |(1ULL <<32 ))),/* INSTRUCTIONS_RETIRED */ -FIXED_EVENT_CONSTRAINT (0x3c , (0x3 |(1ULL <<33 ))),/* UNHALTED_CORE_CYCLES */ +FIXED_EVENT_CONSTRAINT (0x00c0 ,1ULL <<32 ),/* INST_RETIRED.ANY */ +FIXED_EVENT_CONSTRAINT (0x003c ,1ULL <<33 ),/* CPU_CLK_UNHALTED.CORE */ +/* +* FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), CPU_CLK_UNHALTED.REF +* +* Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event +* 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed +* ratio between these counters. +* +* TODO: find/measure the fixed ratio and apply it so that we can +* enable this fixed purpose counter in a transparent way. +*/ INTEL_EVENT_CONSTRAINT (0x10 ,0x1 ),/* FP_COMP_OPS_EXE */ INTEL_EVENT_CONSTRAINT (0x11 ,0x2 ),/* FP_ASSIST */ INTEL_EVENT_CONSTRAINT (0x12 ,0x2 ),/* MUL */
@@ -259,8 +269,9 @@ static struct event_constraint intel_corstatic struct event_constraint intel_nehalem_event_constraints[] = { -FIXED_EVENT_CONSTRAINT (0xc0 , (0xf |(1ULL <<32 ))),/* INSTRUCTIONS_RETIRED */ -FIXED_EVENT_CONSTRAINT (0x3c , (0xf |(1ULL <<33 ))),/* UNHALTED_CORE_CYCLES */ +FIXED_EVENT_CONSTRAINT (0x00c0 ,1ULL <<32 ),/* INST_RETIRED.ANY */ +FIXED_EVENT_CONSTRAINT (0x003c ,1ULL <<33 ),/* CPU_CLK_UNHALTED.CORE */ +FIXED_EVENT_CONSTRAINT (0x013c ,1ULL <<34 ),/* CPU_CLK_UNHALTED.REF */ INTEL_EVENT_CONSTRAINT (0x40 ,0x3 ),/* L1D_CACHE_LD */ INTEL_EVENT_CONSTRAINT (0x41 ,0x3 ),/* L1D_CACHE_ST */ INTEL_EVENT_CONSTRAINT (0x42 ,0x3 ),/* L1D_CACHE_LOCK */
@@ -274,8 +285,9 @@ static struct event_constraint intel_nehstatic struct event_constraint intel_westmere_event_constraints[] = { -FIXED_EVENT_CONSTRAINT (0xc0 , (0xf |(1ULL <<32 ))),/* INSTRUCTIONS_RETIRED */ -FIXED_EVENT_CONSTRAINT (0x3c , (0xf |(1ULL <<33 ))),/* UNHALTED_CORE_CYCLES */ +FIXED_EVENT_CONSTRAINT (0x00c0 ,1ULL <<32 ),/* INST_RETIRED.ANY */ +FIXED_EVENT_CONSTRAINT (0x003c ,1ULL <<33 ),/* CPU_CLK_UNHALTED.CORE */ +FIXED_EVENT_CONSTRAINT (0x013c ,1ULL <<34 ),/* CPU_CLK_UNHALTED.REF */ INTEL_EVENT_CONSTRAINT (0x51 ,0x3 ),/* L1D */ INTEL_EVENT_CONSTRAINT (0x60 ,0x1 ),/* OFFCORE_REQUESTS_OUTSTANDING */ INTEL_EVENT_CONSTRAINT (0x63 ,0x3 ),/* CACHE_LOCK_CYCLES */
@@ -284,8 +296,9 @@ static struct event_constraint intel_wesstatic struct event_constraint intel_gen_event_constraints[] = { -FIXED_EVENT_CONSTRAINT (0xc0 , (0x3 |(1ULL <<32 ))),/* INSTRUCTIONS_RETIRED */ -FIXED_EVENT_CONSTRAINT (0x3c , (0x3 |(1ULL <<33 ))),/* UNHALTED_CORE_CYCLES */ +FIXED_EVENT_CONSTRAINT (0x00c0 ,1ULL <<32 ),/* INST_RETIRED.ANY */ +FIXED_EVENT_CONSTRAINT (0x003c ,1ULL <<33 ),/* CPU_CLK_UNHALTED.CORE */ +FIXED_EVENT_CONSTRAINT (0x013c ,1ULL <<34 ),/* CPU_CLK_UNHALTED.REF */ EVENT_CONSTRAINT_END };
@@ -2602,6 +2615,7 @@ static void __init pmu_check_apic(void)void __init init_hw_perf_events (void ) { +struct event_constraint *c;int err;pr_info ("Performance Events: " );
@@ -2650,6 +2664,14 @@ void __init init_hw_perf_events(void)__EVENT_CONSTRAINT (0 , (1ULL << x86_pmu.num_events) -1 ,0 , x86_pmu.num_events); +for_each_event_constraint (c, x86_pmu.event_constraints) { +if (c->cmask !=INTEL_ARCH_FIXED_MASK ) +continue ; + + c->idxmsk64[0 ] |= (1ULL << x86_pmu.num_events) -1 ; + c->weight += x86_pmu.num_events; + } +pr_info ("... version: %d\n" , x86_pmu.version);pr_info ("... bit width: %d\n" , x86_pmu.event_bits);pr_info ("... generic registers: %d\n" , x86_pmu.num_events);
Peter Zijlstra
maintainerQuoted 23 lines · Peter Zijlstra
+++
linux-2.6/arch/x86/kernel/cpu/perf_event.carch/x86/kernel/cpu/perf_event.c +12
-2@@ -243,8 +243,18 @@ static struct event_constraint intel_corstatic struct event_constraint intel_core2_event_constraints[] = { -FIXED_EVENT_CONSTRAINT (0xc0 , (0x3 |(1ULL <<32 ))),/* INSTRUCTIONS_RETIRED */ -FIXED_EVENT_CONSTRAINT (0x3c , (0x3 |(1ULL <<33 ))),/* UNHALTED_CORE_CYCLES */ +FIXED_EVENT_CONSTRAINT (0x00c0 ,1ULL <<32 ),/* INST_RETIRED.ANY */ +FIXED_EVENT_CONSTRAINT (0x003c ,1ULL <<33 ),/* CPU_CLK_UNHALTED.CORE */ +/* +* FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), CPU_CLK_UNHALTED.REF +* +* Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event +* 0x013c as CPU_CLK_UNHALTED.BUS and specifies there is a fixed +* ratio between these counters. +* +* TODO: find/measure the fixed ratio and apply it so that we can +* enable this fixed purpose counter in a transparent way. +*/ INTEL_EVENT_CONSTRAINT (0x10 ,0x1 ),/* FP_COMP_OPS_EXE */ INTEL_EVENT_CONSTRAINT (0x11 ,0x2 ),/* FP_ASSIST */ INTEL_EVENT_CONSTRAINT (0x12 ,0x2 ),/* MUL */
Quoted 1 line
From what I can measure on the available Core2 systems this ratio is
exactly 1, which would be consistent with the Nehalem and Westmere tables calling this event .REF
Stephane, have you ever observed this ratio to be anything other than 1?
If not, I think we can simply stick this counter back in and not worry about it.
Quoted 24 lines · Peter Zijlstra
Quoted 22 lines · Peter Zijlstra
+++
linux-2.6/arch/x86/kernel/cpu/perf_event.c(quoted diff) +0
-0@@ -243,8 +243,18 @@ static struct event_constraint intel_corstatic struct
event_constraintintel_core2_event_constraints[] =
{
-FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /*INSTRUCTIONS_RETIRED*/
-FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /*UNHALTED_CORE_CYCLES*/
+FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+ /*
+ *FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), CPU_CLK_UNHALTED.REF
+ *
+ * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+ *0x013cas CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+ * ratio between these counters.
+ *
+ * TODO: find/measure the fixed ratio and apply it so that we can
+ * enable this fixed purpose counter in a transparent way.
+ */
INTEL_EVENT_CONSTRAINT(0x10, 0x1), /*FP_COMP_OPS_EXE*/INTEL_EVENT_CONSTRAINT(0x11, 0x2), /*FP_ASSIST*/INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
Quoted 6 lines
Quoted 1 line
From what I can measure on the available Core2 systems this ratio is
exactly 1, which would be consistent with the Nehalem and Westmere tables calling this event .REF
Stephane, have you ever observed this ratio to be anything other than 1?
Using perfmon on Core 2 on a 10s noploop:
pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus
noploop 10
noploop for 10 seconds
23869090125 UNHALTED_REFERENCE_CYCLES
23849336873 UNHALTED_CORE_CYCLES
2652122099 CPU_CLK_UNHALTED:BUS
Quoted 4 lines
If not, I think we can simply stick this counter back in and not worry about it.
Signature · 8 lines
Stephane Eranian | EMEA Software Engineering Google France | 38 avenue de l'Opéra | 75002 Paris Tel : +33 (0) 1 42 68 53 00 This email may be confidential or privileged. If you received this communication by mistake, please don't forward it to anyone else, please erase all copies and attachments, and please let me know that it went to the wrong person. Thanks
Peter Zijlstra
maintainerQuoted 8 lines · Stephane Eranian
Using perfmon on Core 2 on a 10s noploop:
pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus
noploop 10
noploop for 10 seconds
23869090125UNHALTED_REFERENCE_CYCLES
23849336873UNHALTED_CORE_CYCLES
2652122099 CPU_CLK_UNHALTED:BUS
Weird, I used:
while :; do :; done &
while :; do :; done &
while :; do :; done &
while :; do :; done &
perf stat -a -e r013c -e r013c sleep 4
killall bash
Which gives:
Performance counter stats for 'sleep 4':
244235699509090 raw 0x13c
244235695558036 raw 0x13c
4.005485333 seconds time elapsed
And verified it used fixed counter 2 and general purpose counter 0 using sysrq-p.
[523417.108402] CPU#0: gen-PMC0 ctrl: 000000000053013c [523417.108403] CPU#0: gen-PMC0 count: 000000ff80019948 [523417.108405] CPU#0: gen-PMC0 left: 000000007fffffff [523417.108407] CPU#0: gen-PMC1 ctrl: 0000000000000000 [523417.108409] CPU#0: gen-PMC1 count: 0000000000000000 [523417.108411] CPU#0: gen-PMC1 left: 000000007fffb8a8 [523417.108412] CPU#0: fixed-PMC0 count: 0000000000000000 [523417.108414] CPU#0: fixed-PMC1 count: 0000000000000000 [523417.108416] CPU#0: fixed-PMC2 count: 0000010db1db2117
Using -linus, since that doesn't have any of the recent constraint patches in that would avoid us from using fixed-PMC2.
Quoted 20 lines · Peter Zijlstra
Quoted 8 lines · Stephane Eranian
Using perfmon on Core 2 on a 10s noploop:
pfmon -eunhalted_reference_cycles,unhalted_core_cycles,cpu_clk_unhalted:bus noploop 10
noploop for 10 seconds
23869090125UNHALTED_REFERENCE_CYCLES
23849336873UNHALTED_CORE_CYCLES
2652122099 CPU_CLK_UNHALTED:BUSWeird, I used:
while :; do :; done &
while :; do :; done &
while :; do :; done &
while :; do :; done &
perf stat -a -e r013c -e r013c sleep 4
killall bash
Quoted 7 lines
Which gives:
Performance counter stats for 'sleep 4':
244235699509090 raw
0x13c
244235695558036 raw0x13c
Strange. I am running mine on a Q6600.
What's yours?
Quoted 19 lines
4.005485333 seconds time elapsed
And verified it used fixed counter 2 and general purpose counter 0 using sysrq-p.
[523417.108402] CPU#0: gen-PMC0 ctrl: 000000000053013c [523417.108403] CPU#0: gen-PMC0 count: 000000ff80019948 [523417.108405] CPU#0: gen-PMC0 left: 000000007fffffff [523417.108407] CPU#0: gen-PMC1 ctrl: 0000000000000000 [523417.108409] CPU#0: gen-PMC1 count: 0000000000000000 [523417.108411] CPU#0: gen-PMC1 left: 000000007fffb8a8 [523417.108412] CPU#0: fixed-PMC0 count: 0000000000000000 [523417.108414] CPU#0: fixed-PMC1 count: 0000000000000000 [523417.108416] CPU#0: fixed-PMC2 count: 0000010db1db2117Using -linus, since that doesn't have any of the recent constraint patches in that would avoid us from using fixed-PMC2.
Signature · 8 lines
Stephane Eranian | EMEA Software Engineering Google France | 38 avenue de l'Opéra | 75002 Paris Tel : +33 (0) 1 42 68 53 00 This email may be confidential or privileged. If you received this communication by mistake, please don't forward it to anyone else, please erase all copies and attachments, and please let me know that it went to the wrong person. Thanks
Peter Zijlstra
maintainerQuoted 5 lines · Stephane Eranian
Quoted 3 lines
244235699509090 raw
0x13c
244235695558036 raw0x13cStrange. I am running mine on a Q6600.
What's yours?
These numbers came from a T9400, but my Q9450 does something similar.
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 3 lines
244235699509090 raw
0x13c
244235695558036 raw0x13cStrange. I am running mine on a Q6600.
What's yours?These numbers came from a T9400, but my Q9450 does something similar.
Are you sure you are reading from the correct corresponding data registers?
Peter Zijlstra
maintainerQuoted 11 lines · Stephane Eranian
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 3 lines
244235699509090 raw
0x13c
244235695558036 raw0x13cStrange. I am running mine on a Q6600.
What's yours?These numbers came from a T9400, but my Q9450 does something similar.
Are you sure you are reading from the correct corresponding data registers?
It sure got me wondering, I'll go over that fixed counter code in arch/x86/kernel/cpu/perf_event.c in the morning.
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Stephane Eranian
We cannot assume that because
hwc->idx== assign[i], we can avoid reprogramming the counter inhw_perf_enable().The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.
This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.
Quoted 9 lines
Quoted 1 line
Signed-off-by Stephane Eranian <[email protected]>Thanks, got it.
btw, I've also added the below, from what I can make from the docs fixed counter 2 is identical to arch perf event
0x013c, as per table A-1 and A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where0x013cis called CPU_CLK_UNHALTED.BUS.
If you measure 0x013c in a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.
This goes back to an earlier discussion where I was asking about the meaning of the generic PMU events and in particular PERF_COUNT_HW_CPU_CYCLES. Which of the 3 distinct cycle events (unhalted_core_cycles, unhalted_reference_cycles, bus_cycles) does not correspond to?
Quoted 121 lines
---
Subject:perf_events, x86: Fixup fixed counter constraints
From: Peter Zijlstra <[email protected]>
Date: Mon Feb 01 15:36:30 CET 2010Patch
1da53e0230("perf_events, x86: Improve x86 event scheduling") lost us one of the fixed purpose counters and thened8777fc13("perf_events, x86: Fix event constraint masks") broke it even further.Widen the fixed event mask to event+umask and specify the full config for each of the 3 fixed purpose counters. Then let the init code fill out the placement for the GP regs based on the cpuid info.
Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
---
arch/x86/include/asm/perf_event.h | 2 +-arch/x86/kernel/cpu/perf_event.c| 38 ++++++++++++++++++++++++++++++--------2 files changed, 31 insertions(+), 9 deletions(-)Index:
linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================arch/x86/include/asm/perf_event.h +1
-1@@ -50,7 +50,7 @@ INTEL_ARCH_INV_MASK| \ INTEL_ARCH_EDGE_MASK|\ INTEL_ARCH_UNIT_MASK|\ - INTEL_ARCH_EVTSEL_MASK) + INTEL_ARCH_EVENT_MASK) #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)Index:
linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================arch/x86/kernel/cpu/perf_event.c +0
-0@@ -243,8 +243,18 @@ static struct event_constraint intel_corstatic struct
event_constraintintel_core2_event_constraints[] =
{
-FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /*INSTRUCTIONS_RETIRED*/
-FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /*UNHALTED_CORE_CYCLES*/
+FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+ /*
+ *FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), CPU_CLK_UNHALTED.REF
+ *
+ * Core2 has Fixed Counter 2 listed as CPU_CLK_UNHALTED.REF and event
+ *0x013cas CPU_CLK_UNHALTED.BUS and specifies there is a fixed
+ * ratio between these counters.
+ *
+ * TODO: find/measure the fixed ratio and apply it so that we can
+ * enable this fixed purpose counter in a transparent way.
+ */
INTEL_EVENT_CONSTRAINT(0x10,0x1), /*FP_COMP_OPS_EXE*/INTEL_EVENT_CONSTRAINT(0x11, 0x2), /*FP_ASSIST*/INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */arch/x86/kernel/cpu/perf_event.c +0
-0@@ -259,8 +269,9 @@ static struct event_constraint intel_corstatic struct
event_constraintintel_nehalem_event_constraints[] =
{
-FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /*INSTRUCTIONS_RETIRED*/
-FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /*UNHALTED_CORE_CYCLES*/
+FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */INTEL_EVENT_CONSTRAINT(0x40, 0x3), /*L1D_CACHE_LD*/INTEL_EVENT_CONSTRAINT(0x41, 0x3), /*L1D_CACHE_ST*/INTEL_EVENT_CONSTRAINT(0x42, 0x3), /*L1D_CACHE_LOCK*/arch/x86/kernel/cpu/perf_event.c +0
-0@@ -274,8 +285,9 @@ static struct event_constraint intel_nehstatic struct
event_constraintintel_westmere_event_constraints[] = {
-FIXED_EVENT_CONSTRAINT(0xc0, (0xf|(1ULL<<32))), /*INSTRUCTIONS_RETIRED*/
-FIXED_EVENT_CONSTRAINT(0x3c, (0xf|(1ULL<<33))), /*UNHALTED_CORE_CYCLES*/
+FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */INTEL_EVENT_CONSTRAINT(0x60, 0x1), /*OFFCORE_REQUESTS_OUTSTANDING*/INTEL_EVENT_CONSTRAINT(0x63, 0x3), /*CACHE_LOCK_CYCLES*/arch/x86/kernel/cpu/perf_event.c +0
-0@@ -284,8 +296,9 @@ static struct event_constraint intel_wesstatic struct
event_constraintintel_gen_event_constraints[] =
{
-FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /*INSTRUCTIONS_RETIRED*/
-FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /*UNHALTED_CORE_CYCLES*/
+FIXED_EVENT_CONSTRAINT(0x00c0, 1ULL << 32), /* INST_RETIRED.ANY */
+FIXED_EVENT_CONSTRAINT(0x003c, 1ULL << 33), /* CPU_CLK_UNHALTED.CORE */
+FIXED_EVENT_CONSTRAINT(0x013c, 1ULL << 34), /* CPU_CLK_UNHALTED.REF */EVENT_CONSTRAINT_END
};arch/x86/kernel/cpu/perf_event.c +9
-0@@ -2602,6 +2615,7 @@ static void __init pmu_check_apic(void) void __init init_hw_perf_events(void) { + struct event_constraint *c; int err; pr_info("Performance Events: ");@@ -2650,6 +2664,14 @@ void __init init_hw_perf_events(void) __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1, 0, x86_pmu.num_events); + for_each_event_constraint(c, x86_pmu.event_constraints) { + if (c->cmask != INTEL_ARCH_FIXED_MASK) + continue; + + c->idxmsk64[0] |= (1ULL << x86_pmu.num_events) - 1; + c->weight += x86_pmu.num_events; + } + pr_info("... version: %d\n", x86_pmu.version); pr_info("... bit width: %d\n", x86_pmu.event_bits); pr_info("... generic registers: %d\n", x86_pmu.num_events);
Signature · 8 lines
Stephane Eranian | EMEA Software Engineering Google France | 38 avenue de l'Opéra | 75002 Paris Tel : +33 (0) 1 42 68 53 00 This email may be confidential or privileged. If you received this communication by mistake, please don't forward it to anyone else, please erase all copies and attachments, and please let me know that it went to the wrong person. Thanks
Peter Zijlstra
maintainerQuoted 4 lines · Stephane Eranian
Quoted 1 line
btw, I've also added the below, from what I can make from the docs
fixed
Quoted 1 line
counter 2 is identical to arch perf event
0x013c, as per table A-1and
Quoted 8 lines
Quoted 3 lines
A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where
0x013cis called CPU_CLK_UNHALTED.BUS.If you measure
0x013cin a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.
But for Nehalem and Westmere event 0x3c umask 0x01 is referred to as CPU_CLK_UNHALTED.REF_P (Tables A-2 and A-4),
Fixed Counter 2 is referred to as CPU_CLK_UNHALTED.REF (Table A-7).
For Core2 and Atom (Table A-8, A-9) it is called CPU_CLK_UNHALTED.BUS, for these entries there is talk about a fixed ratio.
Quoted 9 lines · Peter Zijlstra
Quoted 8 lines · Stephane Eranian
Quoted 1 line
btw, I've also added the below, from what I can make from the docs
fixed
Quoted 1 line
counter 2 is identical to arch perf event
0x013c, as per table A-1and
Quoted 3 lines
A-7. Both are called CPU_CLK_UNHALTED.REF, except for Core2, where
0x013cis called CPU_CLK_UNHALTED.BUS.
Quoted 8 lines
Quoted 4 lines
If you measure
0x013cin a generic counter or in fixed counter 2 it will count the same thing but not at the same rate. This is true on Core2, Atom, Nehalem, Westmere. The ratio is the clock/bus ratio.But for Nehalem and Westmere event
0x3cumask0x01is referred to as CPU_CLK_UNHALTED.REF_P (Tables A-2 and A-4),
_P means programmable (counter).
on my NHM:
pfmon -v -eunhalted_core_cycles,unhalted_reference_cycles,cpu_clk_unhalted:ref_p
./noploop 10
[FIXED_CTRL(pmc16)=0xaa0 pmi0=1 en0=0x0 any0=0 pmi1=1 en1=0x2 any1=0 pmi2=1 en2=0x2 any2=0] UNHALTED_CORE_CYCLES UNHALTED_REFERENCE_CYCLES [PERFEVTSEL0(pmc0)=0x51013c event_sel=0x3c umask=0x1 os=0 usr=1 anythr=0 en=1 int=1 inv=0 edge=0 cnt_mask=0] CPU_CLK_UNHALTED noploop for 10 seconds
29292225749 UNHALTED_CORE_CYCLES
29292249954 UNHALTED_REFERENCE_CYCLES
1331465398 CPU_CLK_UNHALTED:REF_P
REF_P runs at 133MHz. Core at 2.93GHz.
Quoted 6 lines
Fixed Counter 2 is referred to as CPU_CLK_UNHALTED.REF (Table A-7).
For Core2 and Atom (Table A-8, A-9) it is called CPU_CLK_UNHALTED.BUS, for these entries there is talk about a fixed ratio.
Signature · 8 lines
Stephane Eranian | EMEA Software Engineering Google France | 38 avenue de l'Opéra | 75002 Paris Tel : +33 (0) 1 42 68 53 00 This email may be confidential or privileged. If you received this communication by mistake, please don't forward it to anyone else, please erase all copies and attachments, and please let me know that it went to the wrong person. Thanks
[tip:perf/core] perf_events, x86: Fix bug in hw_perf_enable()
In reply to Stephane EranianCommit-ID: 447a194b393f32699607fd99617a40abd6a95114 Gitweb: http://git.kernel.org/tip/447a194b393f32699607fd99617a40abd6a95114 Author: Stephane Eranian <[email protected]> AuthorDate: Mon, 1 Feb 2010 14:50:01 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 4 Feb 2010 09:59:50 +0100
perf_events, x86: Fix bug in hw_perf_enable()
We cannot assume that because hwc->idx == assign[i], we can avoid reprogramming the counter in hw_perf_enable().
The event may have been scheduled out and another event may have been programmed into this counter. Thus, we need a more robust way of verifying if the counter still contains config/data related to an event.
This patch adds a generation number to each counter on each cpu. Using this mechanism we can verify reliabilty whether the content of a counter corresponds to an event.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 34 ++++++++++++++++++++++++++++------ include/linux/perf_event.h | 2 ++ 2 files changed, 30 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_event.c +28 -6
@@ -90,6 +90,7 @@ struct cpu_hw_events {int n_events;int n_added;int assign[X86_PMC_IDX_MAX ];/* event to counter assignment */ +u64 tags[X86_PMC_IDX_MAX ];struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ };
@@ -1142,6 +1143,8 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->config =ARCH_PERFMON_EVENTSEL_INT ; hwc->idx =-1 ; + hwc->last_cpu =-1 ; + hwc->last_tag = ~0ULL ;/* * Count user and OS events unless requested not to.
@@ -1457,11 +1460,14 @@ static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader,return n; } -static inline void x86_assign_hw_event (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +struct cpu_hw_events *cpuc,int i) { - hwc->idx = idx; +struct hw_perf_event *hwc = &event->hw; + + hwc->idx = cpuc->assign[i]; + hwc->last_cpu =smp_processor_id (); + hwc->last_tag = ++cpuc->tags[i];if (hwc->idx ==X86_PMC_IDX_FIXED_BTS ) { hwc->config_base =0 ;
@@ -1480,6 +1486,15 @@ static inline void x86_assign_hw_event(struct perf_event *event, } } +static inline int match_prev_assignment (struct hw_perf_event *hwc, +struct cpu_hw_events *cpuc, +int i) +{ +return hwc->idx == cpuc->assign[i] && + hwc->last_cpu ==smp_processor_id () && + hwc->last_tag == cpuc->tags[i]; +} +static void __x86_pmu_disable (struct perf_event *event,struct cpu_hw_events *cpuc);void hw_perf_enable (void )
@@ -1508,7 +1523,14 @@ void hw_perf_enable(void) event = cpuc->event_list[i]; hwc = &event->hw; -if (hwc->idx ==-1 || hwc->idx == cpuc->assign[i]) +/* +* we can avoid reprogramming counter if: +* - assigned same counter as last time +* - running on same CPU as last time +* - no other event has used the counter since +*/ +if (hwc->idx ==-1 || +match_prev_assignment (hwc, cpuc, i))continue ;__x86_pmu_disable (event, cpuc);
@@ -1522,12 +1544,12 @@ void hw_perf_enable(void) hwc = &event->hw;if (hwc->idx ==-1 ) { -x86_assign_hw_event (event, hwc, cpuc->assign[i]); +x86_assign_hw_event (event, cpuc, i);x86_perf_event_set_period (event, hwc, hwc->idx); }/* * need to mark as active because x86_pmu_disable() -* clear active_mask and eventsp[] yet it preserves +* clear active_mask and events[] yet it preserves * idx */ set_bit (hwc->idx, cpuc->active_mask);
include/linux/perf_event.h +2 -0
@@ -478,9 +478,11 @@ struct hw_perf_event {union {struct {/* hardware */ u64 config; +u64 last_tag;unsigned long config_base;unsigned long event_base;int idx; +int last_cpu; };struct {/* software */ s64 remaining;