PEBS and LBR support
Subsystems (6)
cover PEBS and LBR support
Simple PEBS and LBR support for perf, this seems to work on the one machine I tested it on (nehalem class).
PATCH 1/14 perf, x86: Remove superfluous arguments to x86_perf_event_set_period()
The second and third argument to x86_perf_event_set_period() are superfluous since they are simple expressions of the first argument. Hence remove them.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 15 +++++++-------- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-)
arch/x86/kernel/cpu/perf_event.c +7 -8
@@ -165,8 +165,7 @@ static DEFINE_PER_CPU(struct cpu_hw_even .enabled = 1, }; -static int x86_perf_event_set_period (struct perf_event *event, -struct hw_perf_event *hwc,int idx); +static int x86_perf_event_set_period (struct perf_event *event);/* * Generalized hw caching related hw_event table, filled
@@ -830,7 +829,7 @@ void hw_perf_enable(void)if (hwc->idx ==-1 ) {x86_assign_hw_event (event, cpuc, i); -x86_perf_event_set_period (event, hwc, hwc->idx); +x86_perf_event_set_period (event); }/* * need to mark as active because x86_pmu_disable()
@@ -871,12 +870,12 @@ static DEFINE_PER_CPU(u64 [X86_PMC_IDX_M* To be called with the event disabled in hw: */ static int -x86_perf_event_set_period (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +x86_perf_event_set_period (struct perf_event *event) { +struct hw_perf_event *hwc = &event->hw;s64 left =atomic64_read (&hwc->period_left);s64 period = hwc->sample_period; -int err, ret =0 ; +int err, ret =0 , idx = hwc->idx;if (idx ==X86_PMC_IDX_FIXED_BTS )return 0 ;
@@ -974,7 +973,7 @@ static int x86_pmu_start(struct perf_eveif (hwc->idx ==-1 )return -EAGAIN ; -x86_perf_event_set_period (event, hwc, hwc->idx); +x86_perf_event_set_period (event); x86_pmu.enable (hwc, hwc->idx);return 0 ;
@@ -1119,7 +1118,7 @@ static int x86_pmu_handle_irq(struct pt_ handled =1 ; data.period = event->hw.last_period; -if (!x86_perf_event_set_period (event, hwc, idx)) +if (!x86_perf_event_set_period (event))continue ;if (perf_event_overflow (event,1 , &data, regs))
arch/x86/kernel/cpu/perf_event_intel.c +1 -1
@@ -700,7 +700,7 @@ static int intel_pmu_save_and_restart(stint ret;x86_perf_event_update (event, hwc, idx); - ret =x86_perf_event_set_period (event, hwc, idx); + ret =x86_perf_event_set_period (event);return ret; }
[tip:perf/urgent] perf, x86: Remove superfluous arguments to x86_perf_event_set_period()
Commit-ID: 07088edb88164c2a2406cd2d9a7be19d8515214b Gitweb: http://git.kernel.org/tip/07088edb88164c2a2406cd2d9a7be19d8515214b Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 2 Mar 2010 20:16:01 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:22:27 +0100
perf, x86: Remove superfluous arguments to x86_perf_event_set_period()
The second and third argument to x86_perf_event_set_period() are superfluous since they are simple expressions of the first argument. Hence remove them.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 15 +++++++-------- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-)
arch/x86/kernel/cpu/perf_event.c +7 -8
@@ -170,8 +170,7 @@ static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; -static int x86_perf_event_set_period (struct perf_event *event, -struct hw_perf_event *hwc,int idx); +static int x86_perf_event_set_period (struct perf_event *event);/* * Generalized hw caching related hw_event table, filled
@@ -835,7 +834,7 @@ void hw_perf_enable(void)if (hwc->idx ==-1 ) {x86_assign_hw_event (event, cpuc, i); -x86_perf_event_set_period (event, hwc, hwc->idx); +x86_perf_event_set_period (event); }/* * need to mark as active because x86_pmu_disable()
@@ -876,12 +875,12 @@ static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);* To be called with the event disabled in hw: */ static int -x86_perf_event_set_period (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +x86_perf_event_set_period (struct perf_event *event) { +struct hw_perf_event *hwc = &event->hw;s64 left =atomic64_read (&hwc->period_left);s64 period = hwc->sample_period; -int err, ret =0 ; +int err, ret =0 , idx = hwc->idx;if (idx ==X86_PMC_IDX_FIXED_BTS )return 0 ;
@@ -979,7 +978,7 @@ static int x86_pmu_start(struct perf_event *event)if (hwc->idx ==-1 )return -EAGAIN ; -x86_perf_event_set_period (event, hwc, hwc->idx); +x86_perf_event_set_period (event); x86_pmu.enable (hwc, hwc->idx);return 0 ;
@@ -1123,7 +1122,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs) handled =1 ; data.period = event->hw.last_period; -if (!x86_perf_event_set_period (event, hwc, idx)) +if (!x86_perf_event_set_period (event))continue ;if (perf_event_overflow (event,1 , &data, regs))
arch/x86/kernel/cpu/perf_event_intel.c +1 -1
@@ -699,7 +699,7 @@ static int intel_pmu_save_and_restart(struct perf_event *event)int ret;x86_perf_event_update (event, hwc, idx); - ret =x86_perf_event_set_period (event, hwc, idx); + ret =x86_perf_event_set_period (event);return ret; }
PATCH 2/14 perf, x86: Remove superfluous arguments to x86_perf_event_update()
In reply to Peter ZijlstraThe second and third argument to x86_perf_event_update() are superfluous since they are simple expressions of the first argument. Hence remove them.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 11 ++++++----- arch/x86/kernel/cpu/perf_event_intel.c | 10 ++-------- 2 files changed, 8 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/perf_event.c +6 -5
@@ -188,11 +188,12 @@ static u64 __read_mostly hw_cache_event_* Returns the delta events processed. */ static u64 -x86_perf_event_update (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +x86_perf_event_update (struct perf_event *event) { +struct hw_perf_event *hwc = &event->hw;int shift =64 - x86_pmu.event_bits;u64 prev_raw_count, new_raw_count; +int idx = hwc->idx;s64 delta;if (idx ==X86_PMC_IDX_FIXED_BTS )
@@ -1059,7 +1060,7 @@ static void x86_pmu_stop(struct perf_eve* Drain the remaining delta count out of a event * that we are disabling: */ -x86_perf_event_update (event, hwc, idx); +x86_perf_event_update (event); cpuc->events[idx] =NULL ; }
@@ -1108,7 +1109,7 @@ static int x86_pmu_handle_irq(struct pt_ event = cpuc->events[idx]; hwc = &event->hw; - val =x86_perf_event_update (event, hwc, idx); + val =x86_perf_event_update (event);if (val & (1ULL << (x86_pmu.event_bits -1 )))continue ;
@@ -1419,7 +1420,7 @@ void __init init_hw_perf_events(void)static inline void x86_pmu_read (struct perf_event *event) { -x86_perf_event_update (event, &event->hw, event->hw.idx); +x86_perf_event_update (event); }static const struct pmu pmu = {
arch/x86/kernel/cpu/perf_event_intel.c +2 -8
@@ -695,14 +695,8 @@ static void intel_pmu_enable_event(struc*/ static int intel_pmu_save_and_restart (struct perf_event *event) { -struct hw_perf_event *hwc = &event->hw; -int idx = hwc->idx; -int ret; - -x86_perf_event_update (event, hwc, idx); - ret =x86_perf_event_set_period (event); - -return ret; +x86_perf_event_update (event); +return x86_perf_event_set_period (event); }static void intel_pmu_reset (void )
[tip:perf/urgent] perf, x86: Remove superfluous arguments to x86_perf_event_update()
Commit-ID: cc2ad4ba8792b9d4ff893ae3b845d2c5a6206fc9 Gitweb: http://git.kernel.org/tip/cc2ad4ba8792b9d4ff893ae3b845d2c5a6206fc9 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 2 Mar 2010 20:18:39 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:22:27 +0100
perf, x86: Remove superfluous arguments to x86_perf_event_update()
The second and third argument to x86_perf_event_update() are superfluous since they are simple expressions of the first argument. Hence remove them.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 11 ++++++----- arch/x86/kernel/cpu/perf_event_intel.c | 10 ++-------- 2 files changed, 8 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/perf_event.c +6 -5
@@ -193,11 +193,12 @@ static u64 __read_mostly hw_cache_event_ids* Returns the delta events processed. */ static u64 -x86_perf_event_update (struct perf_event *event, -struct hw_perf_event *hwc,int idx) +x86_perf_event_update (struct perf_event *event) { +struct hw_perf_event *hwc = &event->hw;int shift =64 - x86_pmu.event_bits;u64 prev_raw_count, new_raw_count; +int idx = hwc->idx;s64 delta;if (idx ==X86_PMC_IDX_FIXED_BTS )
@@ -1064,7 +1065,7 @@ static void x86_pmu_stop(struct perf_event *event)* Drain the remaining delta count out of a event * that we are disabling: */ -x86_perf_event_update (event, hwc, idx); +x86_perf_event_update (event); cpuc->events[idx] =NULL ; }
@@ -1112,7 +1113,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs) event = cpuc->events[idx]; hwc = &event->hw; - val =x86_perf_event_update (event, hwc, idx); + val =x86_perf_event_update (event);if (val & (1ULL << (x86_pmu.event_bits -1 )))continue ;
@@ -1458,7 +1459,7 @@ void __init init_hw_perf_events(void)static inline void x86_pmu_read (struct perf_event *event) { -x86_perf_event_update (event, &event->hw, event->hw.idx); +x86_perf_event_update (event); }static const struct pmu pmu = {
arch/x86/kernel/cpu/perf_event_intel.c +2 -8
@@ -694,14 +694,8 @@ static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)*/ static int intel_pmu_save_and_restart (struct perf_event *event) { -struct hw_perf_event *hwc = &event->hw; -int idx = hwc->idx; -int ret; - -x86_perf_event_update (event, hwc, idx); - ret =x86_perf_event_set_period (event); - -return ret; +x86_perf_event_update (event); +return x86_perf_event_set_period (event); }static void intel_pmu_reset (void )
PATCH 3/14 perf, x86: Change x86_pmu.{enable,disable} calling convention
In reply to Peter ZijlstraPass the full perf_event into the x86_pmu functions so that those may make use of more than the hw_perf_event, and while doing this, remove the superfluous second argument.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 31 +++++++++++++++---------------- arch/x86/kernel/cpu/perf_event_intel.c | 30 +++++++++++++++++------------- arch/x86/kernel/cpu/perf_event_p6.c | 10 ++++++---- 3 files changed, 38 insertions(+), 33 deletions(-)
arch/x86/kernel/cpu/perf_event.c +15 -16
@@ -133,8 +133,8 @@ struct x86_pmu {int (*handle_irq)(struct pt_regs *);void (*disable_all)(void );void (*enable_all)(void ); -void (*enable)(struct hw_perf_event *,int ); -void (*disable)(struct hw_perf_event *,int ); +void (*enable)(struct perf_event *); +void (*disable)(struct perf_event *);unsigned eventsel;unsigned perfctr;u64 (*event_map)(int);
@@ -840,7 +840,7 @@ void hw_perf_enable(void)set_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] = event; - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event);perf_event_update_userpage (event); } cpuc->n_added =0 ;
@@ -853,15 +853,16 @@ void hw_perf_enable(void) x86_pmu.enable_all (); } -static inline void __x86_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static inline void __x86_pmu_enable_event (struct hw_perf_event *hwc) { - (void )checking_wrmsrl (hwc->config_base + idx, + (void )checking_wrmsrl (hwc->config_base + hwc->idx, hwc->config |ARCH_PERFMON_EVENTSEL_ENABLE ); } -static inline void x86_pmu_disable_event (struct hw_perf_event *hwc,int idx) +static inline void x86_pmu_disable_event (struct perf_event *event) { - (void )checking_wrmsrl (hwc->config_base + idx, hwc->config); +struct hw_perf_event *hwc = &event->hw; + (void )checking_wrmsrl (hwc->config_base + hwc->idx, hwc->config); } static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -922,11 +923,11 @@ x86_perf_event_set_period(struct perf_evreturn ret; } -static void x86_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void x86_pmu_enable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);if (cpuc->enabled) -__x86_pmu_enable_event (hwc, idx); +__x86_pmu_enable_event (&event->hw); }/*
@@ -969,13 +970,11 @@ static int x86_pmu_enable(struct perf_evstatic int x86_pmu_start (struct perf_event *event) { -struct hw_perf_event *hwc = &event->hw; - -if (hwc->idx ==-1 ) +if (event->hw.idx ==-1 )return -EAGAIN ;x86_perf_event_set_period (event); - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event);return 0 ; }
@@ -989,7 +988,7 @@ static void x86_pmu_unthrottle(struct pe cpuc->events[hwc->idx] != event))return ; - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event); }void perf_event_print_debug (void )
@@ -1054,7 +1053,7 @@ static void x86_pmu_stop(struct perf_eve* could reenable again: */ clear_bit (idx, cpuc->active_mask); - x86_pmu.disable (hwc, idx); + x86_pmu.disable (event);/* * Drain the remaining delta count out of a event
@@ -1123,7 +1122,7 @@ static int x86_pmu_handle_irq(struct pt_continue ;if (perf_event_overflow (event,1 , &data, regs)) - x86_pmu.disable (hwc, idx); + x86_pmu.disable (event); }if (handled)
arch/x86/kernel/cpu/perf_event_intel.c +17 -13
@@ -548,9 +548,9 @@ static inline void intel_pmu_ack_status( }static inline void -intel_pmu_disable_fixed (struct hw_perf_event *hwc,int __idx) +intel_pmu_disable_fixed (struct hw_perf_event *hwc) { -int idx = __idx -X86_PMC_IDX_FIXED ; +int idx = hwc->idx -X86_PMC_IDX_FIXED ;u64 ctrl_val, mask; mask =0xfULL << (idx *4 );
@@ -622,26 +622,28 @@ static void intel_pmu_drain_bts_buffer(v }static inline void -intel_pmu_disable_event (struct hw_perf_event *hwc,int idx) +intel_pmu_disable_event (struct perf_event *event) { -if (unlikely (idx ==X86_PMC_IDX_FIXED_BTS )) { +struct hw_perf_event *hwc = &event->hw; + +if (unlikely (hwc->idx ==X86_PMC_IDX_FIXED_BTS )) {intel_pmu_disable_bts ();intel_pmu_drain_bts_buffer ();return ; }if (unlikely (hwc->config_base ==MSR_ARCH_PERFMON_FIXED_CTR_CTRL )) { -intel_pmu_disable_fixed (hwc, idx); +intel_pmu_disable_fixed (hwc);return ; } -x86_pmu_disable_event (hwc, idx); +x86_pmu_disable_event (event); }static inline void -intel_pmu_enable_fixed (struct hw_perf_event *hwc,int __idx) +intel_pmu_enable_fixed (struct hw_perf_event *hwc) { -int idx = __idx -X86_PMC_IDX_FIXED ; +int idx = hwc->idx -X86_PMC_IDX_FIXED ;u64 ctrl_val, bits, mask;int err;
@@ -671,9 +673,11 @@ intel_pmu_enable_fixed(struct hw_perf_ev err =checking_wrmsrl (hwc->config_base, ctrl_val); } -static void intel_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void intel_pmu_enable_event (struct perf_event *event) { -if (unlikely (idx ==X86_PMC_IDX_FIXED_BTS )) { +struct hw_perf_event *hwc = &event->hw; + +if (unlikely (hwc->idx ==X86_PMC_IDX_FIXED_BTS )) {if (!__get_cpu_var (cpu_hw_events).enabled)return ;
@@ -682,11 +686,11 @@ static void intel_pmu_enable_event(struc }if (unlikely (hwc->config_base ==MSR_ARCH_PERFMON_FIXED_CTR_CTRL )) { -intel_pmu_enable_fixed (hwc, idx); +intel_pmu_enable_fixed (hwc);return ; } -__x86_pmu_enable_event (hwc, idx); +__x86_pmu_enable_event (hwc); }/*
@@ -774,7 +778,7 @@ again: data.period = event->hw.last_period;if (perf_event_overflow (event,1 , &data, regs)) -intel_pmu_disable_event (&event->hw, bit); +intel_pmu_disable_event (event); }intel_pmu_ack_status (ack);
arch/x86/kernel/cpu/perf_event_p6.c +6 -4
@@ -77,27 +77,29 @@ static void p6_pmu_enable_all(void) }static inline void -p6_pmu_disable_event (struct hw_perf_event *hwc,int idx) +p6_pmu_disable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct hw_perf_event *hwc = &event->hw;u64 val =P6_NOP_EVENT ;if (cpuc->enabled) val |=ARCH_PERFMON_EVENTSEL_ENABLE ; - (void )checking_wrmsrl (hwc->config_base + idx, val); + (void )checking_wrmsrl (hwc->config_base + hwc->idx, val); } -static void p6_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void p6_pmu_enable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct hw_perf_event *hwc = &event->hw;u64 val; val = hwc->config;if (cpuc->enabled) val |=ARCH_PERFMON_EVENTSEL_ENABLE ; - (void )checking_wrmsrl (hwc->config_base + idx, val); + (void )checking_wrmsrl (hwc->config_base + hwc->idx, val); }static __initconst struct x86_pmu p6_pmu = {
[tip:perf/urgent] perf, x86: Change x86_pmu.{enable,disable} calling convention
Commit-ID: aff3d91a913c9ae0c2f56b65b27cbd00c7d27ee3 Gitweb: http://git.kernel.org/tip/aff3d91a913c9ae0c2f56b65b27cbd00c7d27ee3 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 2 Mar 2010 20:32:08 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:22:28 +0100
perf, x86: Change x86_pmu.{enable,disable} calling convention
Pass the full perf_event into the x86_pmu functions so that those may make use of more than the hw_perf_event, and while doing this, remove the superfluous second argument.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 31 +++++++++++++++---------------- arch/x86/kernel/cpu/perf_event_intel.c | 30 +++++++++++++++++------------- arch/x86/kernel/cpu/perf_event_p6.c | 10 ++++++---- 3 files changed, 38 insertions(+), 33 deletions(-)
arch/x86/kernel/cpu/perf_event.c +15 -16
@@ -133,8 +133,8 @@ struct x86_pmu {int (*handle_irq)(struct pt_regs *);void (*disable_all)(void );void (*enable_all)(void ); -void (*enable)(struct hw_perf_event *,int ); -void (*disable)(struct hw_perf_event *,int ); +void (*enable)(struct perf_event *); +void (*disable)(struct perf_event *);unsigned eventsel;unsigned perfctr;u64 (*event_map)(int);
@@ -845,7 +845,7 @@ void hw_perf_enable(void)set_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] = event; - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event);perf_event_update_userpage (event); } cpuc->n_added =0 ;
@@ -858,15 +858,16 @@ void hw_perf_enable(void) x86_pmu.enable_all (); } -static inline void __x86_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static inline void __x86_pmu_enable_event (struct hw_perf_event *hwc) { - (void )checking_wrmsrl (hwc->config_base + idx, + (void )checking_wrmsrl (hwc->config_base + hwc->idx, hwc->config |ARCH_PERFMON_EVENTSEL_ENABLE ); } -static inline void x86_pmu_disable_event (struct hw_perf_event *hwc,int idx) +static inline void x86_pmu_disable_event (struct perf_event *event) { - (void )checking_wrmsrl (hwc->config_base + idx, hwc->config); +struct hw_perf_event *hwc = &event->hw; + (void )checking_wrmsrl (hwc->config_base + hwc->idx, hwc->config); } static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -927,11 +928,11 @@ x86_perf_event_set_period(struct perf_event *event)return ret; } -static void x86_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void x86_pmu_enable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);if (cpuc->enabled) -__x86_pmu_enable_event (hwc, idx); +__x86_pmu_enable_event (&event->hw); }/*
@@ -974,13 +975,11 @@ static int x86_pmu_enable(struct perf_event *event)static int x86_pmu_start (struct perf_event *event) { -struct hw_perf_event *hwc = &event->hw; - -if (hwc->idx ==-1 ) +if (event->hw.idx ==-1 )return -EAGAIN ;x86_perf_event_set_period (event); - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event);return 0 ; }
@@ -994,7 +993,7 @@ static void x86_pmu_unthrottle(struct perf_event *event) cpuc->events[hwc->idx] != event))return ; - x86_pmu.enable (hwc, hwc->idx); + x86_pmu.enable (event); }void perf_event_print_debug (void )
@@ -1059,7 +1058,7 @@ static void x86_pmu_stop(struct perf_event *event)* could reenable again: */ clear_bit (idx, cpuc->active_mask); - x86_pmu.disable (hwc, idx); + x86_pmu.disable (event);/* * Drain the remaining delta count out of a event
@@ -1127,7 +1126,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)continue ;if (perf_event_overflow (event,1 , &data, regs)) - x86_pmu.disable (hwc, idx); + x86_pmu.disable (event); }if (handled)
arch/x86/kernel/cpu/perf_event_intel.c +17 -13
@@ -548,9 +548,9 @@ static inline void intel_pmu_ack_status(u64 ack) }static inline void -intel_pmu_disable_fixed (struct hw_perf_event *hwc,int __idx) +intel_pmu_disable_fixed (struct hw_perf_event *hwc) { -int idx = __idx -X86_PMC_IDX_FIXED ; +int idx = hwc->idx -X86_PMC_IDX_FIXED ;u64 ctrl_val, mask; mask =0xfULL << (idx *4 );
@@ -621,26 +621,28 @@ static void intel_pmu_drain_bts_buffer(void) }static inline void -intel_pmu_disable_event (struct hw_perf_event *hwc,int idx) +intel_pmu_disable_event (struct perf_event *event) { -if (unlikely (idx ==X86_PMC_IDX_FIXED_BTS )) { +struct hw_perf_event *hwc = &event->hw; + +if (unlikely (hwc->idx ==X86_PMC_IDX_FIXED_BTS )) {intel_pmu_disable_bts ();intel_pmu_drain_bts_buffer ();return ; }if (unlikely (hwc->config_base ==MSR_ARCH_PERFMON_FIXED_CTR_CTRL )) { -intel_pmu_disable_fixed (hwc, idx); +intel_pmu_disable_fixed (hwc);return ; } -x86_pmu_disable_event (hwc, idx); +x86_pmu_disable_event (event); }static inline void -intel_pmu_enable_fixed (struct hw_perf_event *hwc,int __idx) +intel_pmu_enable_fixed (struct hw_perf_event *hwc) { -int idx = __idx -X86_PMC_IDX_FIXED ; +int idx = hwc->idx -X86_PMC_IDX_FIXED ;u64 ctrl_val, bits, mask;int err;
@@ -670,9 +672,11 @@ intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx) err =checking_wrmsrl (hwc->config_base, ctrl_val); } -static void intel_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void intel_pmu_enable_event (struct perf_event *event) { -if (unlikely (idx ==X86_PMC_IDX_FIXED_BTS )) { +struct hw_perf_event *hwc = &event->hw; + +if (unlikely (hwc->idx ==X86_PMC_IDX_FIXED_BTS )) {if (!__get_cpu_var (cpu_hw_events).enabled)return ;
@@ -681,11 +685,11 @@ static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx) }if (unlikely (hwc->config_base ==MSR_ARCH_PERFMON_FIXED_CTR_CTRL )) { -intel_pmu_enable_fixed (hwc, idx); +intel_pmu_enable_fixed (hwc);return ; } -__x86_pmu_enable_event (hwc, idx); +__x86_pmu_enable_event (hwc); }/*
@@ -771,7 +775,7 @@ again: data.period = event->hw.last_period;if (perf_event_overflow (event,1 , &data, regs)) -intel_pmu_disable_event (&event->hw, bit); +intel_pmu_disable_event (event); }intel_pmu_ack_status (ack);
arch/x86/kernel/cpu/perf_event_p6.c +6 -4
@@ -77,27 +77,29 @@ static void p6_pmu_enable_all(void) }static inline void -p6_pmu_disable_event (struct hw_perf_event *hwc,int idx) +p6_pmu_disable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct hw_perf_event *hwc = &event->hw;u64 val =P6_NOP_EVENT ;if (cpuc->enabled) val |=ARCH_PERFMON_EVENTSEL_ENABLE ; - (void )checking_wrmsrl (hwc->config_base + idx, val); + (void )checking_wrmsrl (hwc->config_base + hwc->idx, val); } -static void p6_pmu_enable_event (struct hw_perf_event *hwc,int idx) +static void p6_pmu_enable_event (struct perf_event *event) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct hw_perf_event *hwc = &event->hw;u64 val; val = hwc->config;if (cpuc->enabled) val |=ARCH_PERFMON_EVENTSEL_ENABLE ; - (void )checking_wrmsrl (hwc->config_base + idx, val); + (void )checking_wrmsrl (hwc->config_base + hwc->idx, val); }static __initconst struct x86_pmu p6_pmu = {
PATCH 4/14 perf, x86: Use unlocked bitops
In reply to Peter ZijlstraThere is no concurrency on these variables, so don't use LOCK'ed ops.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 8 ++++---- arch/x86/kernel/cpu/perf_event_amd.c | 2 +- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_event.c +4 -4
@@ -638,7 +638,7 @@ static int x86_schedule_events(struct cpif (test_bit (hwc->idx, used_mask))break ; -set_bit (hwc->idx, used_mask); +__set_bit (hwc->idx, used_mask);if (assign) assign[i] = hwc->idx; }
@@ -687,7 +687,7 @@ static int x86_schedule_events(struct cpif (j ==X86_PMC_IDX_MAX )break ; -set_bit (j, used_mask); +__set_bit (j, used_mask);if (assign) assign[i] = j;
@@ -837,7 +837,7 @@ void hw_perf_enable(void)* clear active_mask and events[] yet it preserves * idx */ -set_bit (hwc->idx, cpuc->active_mask); +__set_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] = event; x86_pmu.enable (event);
@@ -1052,7 +1052,7 @@ static void x86_pmu_stop(struct perf_eve* Must be done before we disable, otherwise the nmi handler * could reenable again: */ -clear_bit (idx, cpuc->active_mask); +__clear_bit (idx, cpuc->active_mask); x86_pmu.disable (event);/*
arch/x86/kernel/cpu/perf_event_amd.c +1 -1
@@ -309,7 +309,7 @@ static struct amd_nb *amd_alloc_nb(int c* initialize all possible NB constraints */ for (i =0 ; i < x86_pmu.num_events; i++) { -set_bit (i, nb->event_constraints[i].idxmsk); +__set_bit (i, nb->event_constraints[i].idxmsk); nb->event_constraints[i].weight =1 ; }return nb;
arch/x86/kernel/cpu/perf_event_intel.c +1 -1
@@ -768,7 +768,7 @@ again:for_each_bit (bit, (unsigned long *)&status,X86_PMC_IDX_MAX ) {struct perf_event *event = cpuc->events[bit]; -clear_bit (bit, (unsigned long *) &status); +__clear_bit (bit, (unsigned long *) &status);if (!test_bit (bit, cpuc->active_mask))continue ;
[tip:perf/urgent] perf, x86: Use unlocked bitops
Commit-ID: 34538ee77b39a12702e0f4c3ed9e8fa2dd5eb92c Gitweb: http://git.kernel.org/tip/34538ee77b39a12702e0f4c3ed9e8fa2dd5eb92c Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 2 Mar 2010 21:16:55 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:22:29 +0100
perf, x86: Use unlocked bitops
There is no concurrency on these variables, so don't use LOCK'ed ops.
As to the intel_pmu_handle_irq() status bit clean, nobody uses that so remove it all together.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 8 ++++---- arch/x86/kernel/cpu/perf_event_amd.c | 2 +- arch/x86/kernel/cpu/perf_event_intel.c | 1 - 3 files changed, 5 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_event.c +4 -4
@@ -643,7 +643,7 @@ static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)if (test_bit (hwc->idx, used_mask))break ; -set_bit (hwc->idx, used_mask); +__set_bit (hwc->idx, used_mask);if (assign) assign[i] = hwc->idx; }
@@ -692,7 +692,7 @@ static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)if (j ==X86_PMC_IDX_MAX )break ; -set_bit (j, used_mask); +__set_bit (j, used_mask);if (assign) assign[i] = j;
@@ -842,7 +842,7 @@ void hw_perf_enable(void)* clear active_mask and events[] yet it preserves * idx */ -set_bit (hwc->idx, cpuc->active_mask); +__set_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] = event; x86_pmu.enable (event);
@@ -1057,7 +1057,7 @@ static void x86_pmu_stop(struct perf_event *event)* Must be done before we disable, otherwise the nmi handler * could reenable again: */ -clear_bit (idx, cpuc->active_mask); +__clear_bit (idx, cpuc->active_mask); x86_pmu.disable (event);/*
arch/x86/kernel/cpu/perf_event_amd.c +1 -1
@@ -287,7 +287,7 @@ static struct amd_nb *amd_alloc_nb(int cpu, int nb_id)* initialize all possible NB constraints */ for (i =0 ; i < x86_pmu.num_events; i++) { -set_bit (i, nb->event_constraints[i].idxmsk); +__set_bit (i, nb->event_constraints[i].idxmsk); nb->event_constraints[i].weight =1 ; }return nb;
arch/x86/kernel/cpu/perf_event_intel.c +0 -1
@@ -765,7 +765,6 @@ again:for_each_set_bit (bit, (unsigned long *)&status,X86_PMC_IDX_MAX ) {struct perf_event *event = cpuc->events[bit]; -clear_bit (bit, (unsigned long *) &status);if (!test_bit (bit, cpuc->active_mask))continue ;
PATCH 5/14 perf: Generic perf_sample_data initialization
In reply to Peter ZijlstraThis makes it easier to extend perf_sample_data and fixes a bug on arm and sparc, which failed to set ->raw to NULL, which can cause crashes when combined with PERF_SAMPLE_RAW.
It also optimizes PowerPC and tracepoint, because the struct initialization is forced to zero out the whole structure.
LKML-Reference: <new-submission>
--- arch/arm/kernel/perf_event.c | 4 ++-- arch/powerpc/kernel/perf_event.c | 8 ++++---- arch/sparc/kernel/perf_event.c | 2 +- arch/x86/kernel/cpu/perf_event.c | 3 +-- arch/x86/kernel/cpu/perf_event_intel.c | 6 ++---- include/linux/perf_event.h | 7 +++++++ kernel/perf_event.c | 21 ++++++++------------- 7 files changed, 25 insertions(+), 26 deletions(-)
arch/arm/kernel/perf_event.c +2 -2
@@ -965,7 +965,7 @@ armv6pmu_handle_irq(int irq_num,*/ armv6_pmcr_write (pmcr); - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);for (idx =0 ; idx <= armpmu->num_events; ++idx) {
@@ -1945,7 +1945,7 @@ static irqreturn_t armv7pmu_handle_irq(i*/ regs =get_irq_regs (); - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);for (idx =0 ; idx <= armpmu->num_events; ++idx) {
arch/powerpc/kernel/perf_event.c +4 -4
@@ -1164,10 +1164,10 @@ static void record_and_restart(struct pe* Finally record data if requested. */ if (record) { -struct perf_sample_data data = { - .addr = ~0ULL , - .period = event->hw.last_period, - }; +struct perf_sample_data data; + +perf_sample_data_init (&data, ~0ULL ); + data.period = event->hw.last_period;if (event->attr.sample_type &PERF_SAMPLE_ADDR )perf_get_data_addr (regs, &data.addr);
arch/sparc/kernel/perf_event.c +1 -1
@@ -1189,7 +1189,7 @@ static int __kprobes perf_event_nmi_hand regs = args->regs; - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
arch/x86/kernel/cpu/perf_event.c +1 -2
@@ -1096,8 +1096,7 @@ static int x86_pmu_handle_irq(struct pt_int idx, handled =0 ;u64 val; - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
arch/x86/kernel/cpu/perf_event_intel.c +2 -4
@@ -590,10 +590,9 @@ static void intel_pmu_drain_bts_buffer(v ds->bts_index = ds->bts_buffer_base; +perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; - data.addr =0 ; - data.raw =NULL ; regs.ip =0 ;/*
@@ -740,8 +739,7 @@ static int intel_pmu_handle_irq(struct pint bit, loops;u64 ack, status; - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
include/linux/perf_event.h +7 -0
@@ -801,6 +801,13 @@ struct perf_sample_data {struct perf_raw_record *raw; }; +static inline +void perf_sample_data_init (struct perf_sample_data *data,u64 addr) +{ + data->addr = addr; + data->raw =NULL ; +} + extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data,
kernel/perf_event.c +8 -13
@@ -4108,8 +4108,7 @@ void __perf_sw_event(u32 event_id, u64 nif (rctx <0 )return ; - data.addr = addr; - data.raw =NULL ; +perf_sample_data_init (&data, addr);do_perf_sw_event (PERF_TYPE_SOFTWARE , event_id, nr, nmi, &data, regs);
@@ -4154,11 +4153,10 @@ static enum hrtimer_restart perf_sweventstruct perf_event *event;u64 period; - event = container_of(hrtimer, struct perf_event, hw.hrtimer); + event = container_of(hrtimer, struct perf_event, hw.hrtimer); event->pmu->read (event); - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; regs =get_irq_regs ();/*
@@ -4322,17 +4320,15 @@ static const struct pmu perf_ops_task_clvoid perf_tp_event (int event_id,u64 addr,u64 count,void *record,int entry_size) { +struct pt_regs *regs =get_irq_regs (); +struct perf_sample_data data;struct perf_raw_record raw = { .size = entry_size, .data = record, }; -struct perf_sample_data data = { - .addr = addr, - .raw = &raw, - }; - -struct pt_regs *regs =get_irq_regs (); +perf_sample_data_init (&data, addr); + data.raw = &raw;if (!regs) regs =task_pt_regs (current);
@@ -4448,8 +4444,7 @@ void perf_bp_event(struct perf_event *bpstruct perf_sample_data sample;struct pt_regs *regs = data; - sample.raw =NULL ; - sample.addr = bp->attr.bp_addr; +perf_sample_data_init (&sample, bp->attr.bp_addr);if (!perf_exclude_event (bp, regs))perf_swevent_add (bp,1 ,1 , &sample, regs);
[tip:perf/urgent] perf: Provide generic perf_sample_data initialization
Commit-ID: dc1d628a67a8f042e711ea5accc0beedc3ef0092 Gitweb: http://git.kernel.org/tip/dc1d628a67a8f042e711ea5accc0beedc3ef0092 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 3 Mar 2010 15:55:04 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:22:23 +0100
perf: Provide generic perf_sample_data initialization
This makes it easier to extend perf_sample_data and fixes a bug on arm and sparc, which failed to set ->raw to NULL, which can cause crashes when combined with PERF_SAMPLE_RAW.
It also optimizes PowerPC and tracepoint, because the struct initialization is forced to zero out the whole structure.
LKML-Reference: <[email protected]>
--- arch/arm/kernel/perf_event.c | 4 ++-- arch/powerpc/kernel/perf_event.c | 8 ++++---- arch/sparc/kernel/perf_event.c | 2 +- arch/x86/kernel/cpu/perf_event.c | 3 +-- arch/x86/kernel/cpu/perf_event_intel.c | 6 ++---- include/linux/perf_event.h | 7 +++++++ kernel/perf_event.c | 21 ++++++++------------- 7 files changed, 25 insertions(+), 26 deletions(-)
arch/arm/kernel/perf_event.c +2 -2
@@ -965,7 +965,7 @@ armv6pmu_handle_irq(int irq_num,*/ armv6_pmcr_write (pmcr); - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);for (idx =0 ; idx <= armpmu->num_events; ++idx) {
@@ -1945,7 +1945,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)*/ regs =get_irq_regs (); - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);for (idx =0 ; idx <= armpmu->num_events; ++idx) {
arch/powerpc/kernel/perf_event.c +4 -4
@@ -1164,10 +1164,10 @@ static void record_and_restart(struct perf_event *event, unsigned long val,* Finally record data if requested. */ if (record) { -struct perf_sample_data data = { - .addr = ~0ULL , - .period = event->hw.last_period, - }; +struct perf_sample_data data; + +perf_sample_data_init (&data, ~0ULL ); + data.period = event->hw.last_period;if (event->attr.sample_type &PERF_SAMPLE_ADDR )perf_get_data_addr (regs, &data.addr);
arch/sparc/kernel/perf_event.c +1 -1
@@ -1189,7 +1189,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, regs = args->regs; - data.addr =0 ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
arch/x86/kernel/cpu/perf_event.c +1 -2
@@ -1097,8 +1097,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)int idx, handled =0 ;u64 val; - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
arch/x86/kernel/cpu/perf_event_intel.c +2 -4
@@ -590,10 +590,9 @@ static void intel_pmu_drain_bts_buffer(void) ds->bts_index = ds->bts_buffer_base; +perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; - data.addr =0 ; - data.raw =NULL ; regs.ip =0 ;/*
@@ -742,8 +741,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)int bit, loops;u64 ack, status; - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); cpuc = &__get_cpu_var (cpu_hw_events);
include/linux/perf_event.h +7 -0
@@ -801,6 +801,13 @@ struct perf_sample_data {struct perf_raw_record *raw; }; +static inline +void perf_sample_data_init (struct perf_sample_data *data,u64 addr) +{ + data->addr = addr; + data->raw =NULL ; +} + extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data,
kernel/perf_event.c +8 -13
@@ -4108,8 +4108,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi,if (rctx <0 )return ; - data.addr = addr; - data.raw =NULL ; +perf_sample_data_init (&data, addr);do_perf_sw_event (PERF_TYPE_SOFTWARE , event_id, nr, nmi, &data, regs);
@@ -4154,11 +4153,10 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)struct perf_event *event;u64 period; - event = container_of(hrtimer, struct perf_event, hw.hrtimer); + event = container_of(hrtimer, struct perf_event, hw.hrtimer); event->pmu->read (event); - data.addr =0 ; - data.raw =NULL ; +perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; regs =get_irq_regs ();/*
@@ -4322,17 +4320,15 @@ static const struct pmu perf_ops_task_clock = {void perf_tp_event (int event_id,u64 addr,u64 count,void *record,int entry_size) { +struct pt_regs *regs =get_irq_regs (); +struct perf_sample_data data;struct perf_raw_record raw = { .size = entry_size, .data = record, }; -struct perf_sample_data data = { - .addr = addr, - .raw = &raw, - }; - -struct pt_regs *regs =get_irq_regs (); +perf_sample_data_init (&data, addr); + data.raw = &raw;if (!regs) regs =task_pt_regs (current);
@@ -4448,8 +4444,7 @@ void perf_bp_event(struct perf_event *bp, void *data)struct perf_sample_data sample;struct pt_regs *regs = data; - sample.raw =NULL ; - sample.addr = bp->attr.bp_addr; +perf_sample_data_init (&sample, bp->attr.bp_addr);if (!perf_exclude_event (bp, regs))perf_swevent_add (bp,1 ,1 , &sample, regs);
PATCH 6/14 perf, x86: PEBS infrastructure
In reply to Peter ZijlstraImplement a simple PEBS model that always takes a single PEBS event at a time. This is done so that the interaction with the rest of the system is as expected (freq adjust, period randomization, lbr).
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 223 ++++------- arch/x86/kernel/cpu/perf_event_intel.c | 152 +------- arch/x86/kernel/cpu/perf_event_intel_ds.c | 567 ++++++++++++++++++++++++++++++ include/linux/perf_event.h | 3 4 files changed, 682 insertions(+), 263 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_ds.c new file +567 -0
@@ -0,0 +1,567 @@ +#ifdef CONFIG_CPU_SUP_INTEL + +/* The maximal number of PEBS events: */ +#define MAX_PEBS_EVENTS 4 + +/* The size of a BTS record in bytes: */ +#define BTS_RECORD_SIZE 24 + +#define BTS_BUFFER_SIZE (PAGE_SIZE << 4) +#define PEBS_BUFFER_SIZE PAGE_SIZE + +/* +* pebs_record_32 for p4 and core not supported + +struct pebs_record_32 { +u32 flags, ip; +u32 ax, bc, cx, dx; +u32 si, di, bp, sp; +}; + +*/ + +struct pebs_record_core { +u64 flags, ip; +u64 ax, bx, cx, dx; +u64 si, di, bp, sp; +u64 r8, r9, r10, r11; +u64 r12, r13, r14, r15; +}; + +struct pebs_record_nhm { +u64 flags, ip; +u64 ax, bx, cx, dx; +u64 si, di, bp, sp; +u64 r8, r9, r10, r11; +u64 r12, r13, r14, r15; +u64 status, dla, dse, lat; +}; + +/* +* Bits in the debugctlmsr controlling branch tracing. +*/ +#define X86_DEBUGCTL_TR (1 << 6) +#define X86_DEBUGCTL_BTS (1 << 7) +#define X86_DEBUGCTL_BTINT (1 << 8) +#define X86_DEBUGCTL_BTS_OFF_OS (1 << 9) +#define X86_DEBUGCTL_BTS_OFF_USR (1 << 10) + +/* +* A debug store configuration. +* +* We only support architectures that use 64bit fields. +*/ +struct debug_store { +u64 bts_buffer_base; +u64 bts_index; +u64 bts_absolute_maximum; +u64 bts_interrupt_threshold; +u64 pebs_buffer_base; +u64 pebs_index; +u64 pebs_absolute_maximum; +u64 pebs_interrupt_threshold; +u64 pebs_event_reset[MAX_PEBS_EVENTS ]; +}; + +static inline void init_debug_store_on_cpu (int cpu) +{ +struct debug_store *ds =per_cpu (cpu_hw_events, cpu).ds; + +if (!ds) +return ; + +wrmsr_on_cpu (cpu,MSR_IA32_DS_AREA , + (u32)((u64 )(unsigned long )ds), + (u32)((u64 )(unsigned long )ds >>32 )); +} + +static inline void fini_debug_store_on_cpu (int cpu) +{ +if (!per_cpu (cpu_hw_events, cpu).ds) +return ; + +wrmsr_on_cpu (cpu,MSR_IA32_DS_AREA ,0 ,0 ); +} + +static void release_ds_buffers (void ) +{ +int cpu; + +if (!x86_pmu.bts && !x86_pmu.pebs) +return ; + +get_online_cpus (); + +for_each_online_cpu (cpu) +fini_debug_store_on_cpu (cpu); + +for_each_possible_cpu (cpu) { +struct debug_store *ds =per_cpu (cpu_hw_events, cpu).ds; + +if (!ds) +continue ; + +per_cpu (cpu_hw_events, cpu).ds =NULL ; + +kfree ((void *)(unsigned long )ds->pebs_buffer_base); +kfree ((void *)(unsigned long )ds->bts_buffer_base); +kfree (ds); + } + +put_online_cpus (); +} + +static int reserve_ds_buffers (void ) +{ +int cpu, err =0 ; + +if (!x86_pmu.bts && !x86_pmu.pebs) +return 0 ; + +get_online_cpus (); + +for_each_possible_cpu (cpu) { +struct debug_store *ds; +void *buffer; +int max, thresh; + + err = -ENOMEM ; + ds =kzalloc (sizeof (*ds),GFP_KERNEL ); +if (unlikely (!ds)) { +kfree (buffer); +break ; + } +per_cpu (cpu_hw_events, cpu).ds = ds; + +if (x86_pmu.bts) { + buffer =kzalloc (BTS_BUFFER_SIZE ,GFP_KERNEL ); +if (unlikely (!buffer)) +break ; + + max =BTS_BUFFER_SIZE /BTS_RECORD_SIZE ; + thresh = max /16 ; + + ds->bts_buffer_base = (u64 )(unsigned long )buffer; + ds->bts_index = ds->bts_buffer_base; + ds->bts_absolute_maximum = ds->bts_buffer_base + + max *BTS_RECORD_SIZE ; + ds->bts_interrupt_threshold = ds->bts_absolute_maximum - + thresh *BTS_RECORD_SIZE ; + } + +if (x86_pmu.pebs) { + buffer =kzalloc (PEBS_BUFFER_SIZE ,GFP_KERNEL ); +if (unlikely (!buffer)) +break ; + + max =PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size; + + ds->pebs_buffer_base = (u64 )(unsigned long )buffer; + ds->pebs_index = ds->pebs_buffer_base; + ds->pebs_absolute_maximum = ds->pebs_buffer_base + + max * x86_pmu.pebs_record_size; +/* +* Always use single record PEBS +*/ + ds->pebs_interrupt_threshold = ds->pebs_buffer_base + + x86_pmu.pebs_record_size; + } + + err =0 ; + } + +if (err) +release_ds_buffers (); +else { +for_each_online_cpu (cpu) +init_debug_store_on_cpu (cpu); + } + +put_online_cpus (); + +return err; +} + +/* +* BTS +*/ + +static struct event_constraint bts_constraint = +EVENT_CONSTRAINT (0 ,1ULL <<X86_PMC_IDX_FIXED_BTS ,0 ); + +static void intel_pmu_enable_bts (u64 config) +{ +unsigned long debugctlmsr; + + debugctlmsr =get_debugctlmsr (); + + debugctlmsr |=X86_DEBUGCTL_TR ; + debugctlmsr |=X86_DEBUGCTL_BTS ; + debugctlmsr |=X86_DEBUGCTL_BTINT ; + +if (!(config &ARCH_PERFMON_EVENTSEL_OS )) + debugctlmsr |=X86_DEBUGCTL_BTS_OFF_OS ; + +if (!(config &ARCH_PERFMON_EVENTSEL_USR )) + debugctlmsr |=X86_DEBUGCTL_BTS_OFF_USR ; + +update_debugctlmsr (debugctlmsr); +} + +static void intel_pmu_disable_bts (void ) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +unsigned long debugctlmsr; + +if (!cpuc->ds) +return ; + + debugctlmsr =get_debugctlmsr (); + + debugctlmsr &= + ~(X86_DEBUGCTL_TR |X86_DEBUGCTL_BTS |X86_DEBUGCTL_BTINT | +X86_DEBUGCTL_BTS_OFF_OS |X86_DEBUGCTL_BTS_OFF_USR ); + +update_debugctlmsr (debugctlmsr); +} + +static void intel_pmu_drain_bts_buffer (void ) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct debug_store *ds = cpuc->ds; +struct bts_record { +u64 from; +u64 to; +u64 flags; + }; +struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS ]; +struct bts_record *at, *top; +struct perf_output_handle handle; +struct perf_event_header header; +struct perf_sample_data data; +struct pt_regs regs; + +if (!event) +return ; + +if (!ds) +return ; + + at = (struct bts_record *)(unsigned long )ds->bts_buffer_base; + top = (struct bts_record *)(unsigned long )ds->bts_index; + +if (top <= at) +return ; + + ds->bts_index = ds->bts_buffer_base; + +perf_sample_data_init (&data,0 ); + data.period = event->hw.last_period; + regs.ip =0 ; + +/* +* Prepare a generic sample, i.e. fill in the invariant fields. +* We will overwrite the from and to address before we output +* the sample. +*/ +perf_prepare_sample (&header, &data, event, ®s); + +if (perf_output_begin (&handle, event, header.size * (top - at),1 ,1 )) +return ; + +for (; at < top; at++) { + data.ip = at->from; + data.addr = at->to; + +perf_output_sample (&handle, &header, &data, event); + } + +perf_output_end (&handle); + +/* There's new data available. */ + event->hw.interrupts++; + event->pending_kill =POLL_IN ; +} + +/* +* PEBS +*/ + +static struct event_constraint intel_core_pebs_events[] = { +PEBS_EVENT_CONSTRAINT (0x00c0 ,0x1 ),/* INSTR_RETIRED.ANY */ +PEBS_EVENT_CONSTRAINT (0xfec1 ,0x1 ),/* X87_OPS_RETIRED.ANY */ +PEBS_EVENT_CONSTRAINT (0x00c5 ,0x1 ),/* BR_INST_RETIRED.MISPRED */ +PEBS_EVENT_CONSTRAINT (0x1fc7 ,0x1 ),/* SIMD_INST_RETURED.ANY */ +PEBS_EVENT_CONSTRAINT (0x01cb ,0x1 ),/* MEM_LOAD_RETIRED.L1D_MISS */ +PEBS_EVENT_CONSTRAINT (0x02cb ,0x1 ),/* MEM_LOAD_RETIRED.L1D_LINE_MISS */ +PEBS_EVENT_CONSTRAINT (0x04cb ,0x1 ),/* MEM_LOAD_RETIRED.L2_MISS */ +PEBS_EVENT_CONSTRAINT (0x08cb ,0x1 ),/* MEM_LOAD_RETIRED.L2_LINE_MISS */ +PEBS_EVENT_CONSTRAINT (0x10cb ,0x1 ),/* MEM_LOAD_RETIRED.DTLB_MISS */ +EVENT_CONSTRAINT_END +}; + +static struct event_constraint intel_nehalem_pebs_events[] = { +PEBS_EVENT_CONSTRAINT (0x00c0 ,0xf ),/* INSTR_RETIRED.ANY */ +PEBS_EVENT_CONSTRAINT (0xfec1 ,0xf ),/* X87_OPS_RETIRED.ANY */ +PEBS_EVENT_CONSTRAINT (0x00c5 ,0xf ),/* BR_INST_RETIRED.MISPRED */ +PEBS_EVENT_CONSTRAINT (0x1fc7 ,0xf ),/* SIMD_INST_RETURED.ANY */ +PEBS_EVENT_CONSTRAINT (0x01cb ,0xf ),/* MEM_LOAD_RETIRED.L1D_MISS */ +PEBS_EVENT_CONSTRAINT (0x02cb ,0xf ),/* MEM_LOAD_RETIRED.L1D_LINE_MISS */ +PEBS_EVENT_CONSTRAINT (0x04cb ,0xf ),/* MEM_LOAD_RETIRED.L2_MISS */ +PEBS_EVENT_CONSTRAINT (0x08cb ,0xf ),/* MEM_LOAD_RETIRED.L2_LINE_MISS */ +PEBS_EVENT_CONSTRAINT (0x10cb ,0xf ),/* MEM_LOAD_RETIRED.DTLB_MISS */ +EVENT_CONSTRAINT_END +}; + +static struct event_constraint * +intel_pebs_constraints (struct perf_event *event) +{ +struct event_constraint *c; + +if (!event->attr.precise) +return NULL ; + +if (x86_pmu.pebs_constraints) { +for_each_event_constraint (c, x86_pmu.pebs_constraints) { +if ((event->hw.config & c->cmask) == c->code) +return c; + } + } + +return &emptyconstraint; +} + +static void intel_pmu_pebs_enable (struct hw_perf_event *hwc) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +u64 val = cpuc->pebs_enabled; + + hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT ; + + val |=1ULL << hwc->idx; +wrmsrl (MSR_IA32_PEBS_ENABLE , val); +} + +static void intel_pmu_pebs_disable (struct hw_perf_event *hwc) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +u64 val = cpuc->pebs_enabled; + + val &= ~(1ULL << hwc->idx); +wrmsrl (MSR_IA32_PEBS_ENABLE , val); + + hwc->config |=ARCH_PERFMON_EVENTSEL_INT ; +} + +static void intel_pmu_pebs_enable_all (void ) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); + +if (cpuc->pebs_enabled) +wrmsrl (MSR_IA32_PEBS_ENABLE , cpuc->pebs_enabled); +} + +static void intel_pmu_pebs_disable_all (void ) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); + +if (cpuc->pebs_enabled) +wrmsrl (MSR_IA32_PEBS_ENABLE ,0 ); +} + +static int intel_pmu_save_and_restart (struct perf_event *event); +static void intel_pmu_disable_event (struct perf_event *event); + +static void intel_pmu_drain_pebs_core (struct pt_regs *iregs) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct debug_store *ds = cpuc->ds; +struct perf_event *event = cpuc->events[0 ];/* PMC0 only */ +struct pebs_record_core *at, *top; +struct perf_sample_data data; +struct pt_regs regs; +int n; + +if (!event || !ds || !x86_pmu.pebs) +return ; + +intel_pmu_pebs_disable_all (); + + at = (struct pebs_record_core *)(unsigned long )ds->pebs_buffer_base; + top = (struct pebs_record_core *)(unsigned long )ds->pebs_index; + +if (top <= at) + goto out; + + ds->pebs_index = ds->pebs_buffer_base; + +if (!intel_pmu_save_and_restart (event)) + goto out; + +perf_sample_data_init (&data,0 ); + data.period = event->hw.last_period; + + n = top - at; + +/* +* Should not happen, we program the threshold at 1 and do not +* set a reset value. +*/ +if (unlikely (n >1 )) { +trace_printk ("PEBS: too many events: %d\n" , n); + at += n-1 ; + } + +/* +* We use the interrupt regs as a base because the PEBS record +* does not contain a full regs set, specifically it seems to +* lack segment descriptors, which get used by things like +* user_mode(). +* +* In the simple case fix up only the IP and BP,SP regs, for +* PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly. +* A possible PERF_SAMPLE_REGS will have to transfer all regs. +*/ + regs = *iregs; + regs.ip = at->ip; + regs.bp = at->bp; + regs.sp = at->sp; + +if (perf_event_overflow (event,1 , &data, ®s)) +intel_pmu_disable_event (event); + +out: +intel_pmu_pebs_enable_all (); +} + +static void intel_pmu_drain_pebs_nhm (struct pt_regs *iregs) +{ +struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +struct debug_store *ds = cpuc->ds; +struct pebs_record_nhm *at, *top; +struct perf_sample_data data; +struct perf_event *event =NULL ; +struct pt_regs regs; +int bit, n; + +if (!ds || !x86_pmu.pebs) +return ; + +intel_pmu_pebs_disable_all (); + + at = (struct pebs_record_nhm *)(unsigned long )ds->pebs_buffer_base; + top = (struct pebs_record_nhm *)(unsigned long )ds->pebs_index; + +if (top <= at) + goto out; + + ds->pebs_index = ds->pebs_buffer_base; + + n = top - at; + +/* +* Should not happen, we program the threshold at 1 and do not +* set a reset value. +*/ +if (unlikely (n >MAX_PEBS_EVENTS )) +trace_printk ("PEBS: too many events: %d\n" , n); + +for ( ; at < top; at++) { +for_each_bit (bit, (unsigned long *)&at->status,MAX_PEBS_EVENTS ) { +if (!cpuc->events[bit]->attr.precise) +continue ; + +if (event) +trace_printk ("PEBS: status: %Lx\n" , at->status); + + event = cpuc->events[bit]; + } + +if (!event) { +trace_printk ("PEBS: interrupt, status: %Lx\n" , + at->status); +continue ; + } + +if (!intel_pmu_save_and_restart (event)) +continue ; + +perf_sample_data_init (&data,0 ); + data.period = event->hw.last_period; + +/* +* See the comment in intel_pmu_drain_pebs_core() +*/ + regs = *iregs; + regs.ip = at->ip; + regs.bp = at->bp; + regs.sp = at->sp; + +if (perf_event_overflow (event,1 , &data, ®s)) +intel_pmu_disable_event (event); + } +out: +intel_pmu_pebs_enable_all (); +} + +/* +* BTS, PEBS probe and setup +*/ + +static void intel_ds_init (void ) +{ +/* +* No support for 32bit formats +*/ +if (!boot_cpu_has (X86_FEATURE_DTES64 )) +return ; + + x86_pmu.bts =boot_cpu_has (X86_FEATURE_BTS ); + x86_pmu.pebs =boot_cpu_has (X86_FEATURE_PEBS ); +if (x86_pmu.pebs) { +int format =0 ; + +if (x86_pmu.version >1 ) { +u64 capabilities; +/* +* v2+ has a PEBS format field +*/ +rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); + format = (capabilities >>8 ) &0xf ; + } + +switch (format) { +case 0 : +printk (KERN_CONT "PEBS v0, " ); + x86_pmu.pebs_record_size =sizeof (struct pebs_record_core ); + x86_pmu.drain_pebs = intel_pmu_drain_pebs_core; + x86_pmu.pebs_constraints = intel_core_pebs_events; +break ; + +case 1 : +printk (KERN_CONT "PEBS v1, " ); + x86_pmu.pebs_record_size =sizeof (struct pebs_record_nhm ); + x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm; + x86_pmu.pebs_constraints = intel_nehalem_pebs_events; +break ; + +default : +printk (KERN_CONT "PEBS unknown format: %d, " , format); + x86_pmu.pebs =0 ; +break ; + } + } +} + +#else /* CONFIG_CPU_SUP_INTEL */ + +static int reseve_ds_buffers (void ) +{ +return 0 ; +} + +static void release_ds_buffers (void ) +{ +} + +#endif /* CONFIG_CPU_SUP_INTEL */
arch/x86/kernel/cpu/perf_event.c +78 -145
334 lines not on this page · raw message
arch/x86/kernel/cpu/perf_event_intel.c +35 -117
234 lines not on this page · raw message
include/linux/perf_event.h +2 -1
@@ -203,8 +203,9 @@ struct perf_event_attr { enable_on_exec : 1, /* next exec enables */ task : 1, /* trace fork/exit */ watermark : 1, /* wakeup_watermark */ + precise : 1, /* OoO invariant counter */ - __reserved_1 : 49; + __reserved_1 : 48;union {__u32 wakeup_events;/* wakeup every n events */
Re: [PATCH 06/14] perf, x86: PEBS infrastructure
Quoted 7 lines · Peter Zijlstra
Implement a simple PEBS model that always takes a single PEBS event at a time. This is done so that the interaction with the rest of the system is as expected (freq adjust, period randomization, lbr).
Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
---
...
include/linux/perf_event.h struct perf_event_attr { · line 206
Quoted 5 lines
include/linux/perf_event.h +1
-0@@ -203,8 +203,9 @@ struct perf_event_attr { enable_on_exec : 1, /* next exec enables */ task : 1, /* trace fork/exit */ watermark : 1, /* wakeup_watermark */ + precise : 1, /* OoO invariant counter */
Could you explain in a bit more detail what this means?
Also, it would be good to mention the ABI addition in the patch description, and explain it briefly there.
Paul.
Peter Zijlstra
maintainerRe: [PATCH 06/14] perf, x86: PEBS infrastructure
Quoted 20 lines · Paul Mackerras
Quoted 7 lines · Peter Zijlstra
Implement a simple PEBS model that always takes a single PEBS event at a time. This is done so that the interaction with the rest of the system is as expected (freq adjust, period randomization, lbr).
Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
---...
include/linux/perf_event.h struct perf_event_attr { · line 206
Quoted 5 lines
include/linux/perf_event.h +1
-0@@ -203,8 +203,9 @@ struct perf_event_attr { enable_on_exec : 1, /* next exec enables */ task : 1, /* trace fork/exit */ watermark : 1, /* wakeup_watermark */ + precise : 1, /* OoO invariant counter */Could you explain in a bit more detail what this means?
Quoted 2 lines
Also, it would be good to mention the ABI addition in the patch description, and explain it briefly there.
Quite so, my bad.
So on Intel regular PMIs can happen several instructions later than the actual event due to out-of-order processing of the instruction stream, that is, it doesn't keep the IP of the actual instruction that triggered the event, so all we have is the IP of where the interrupt happened (the difference between these IPs is called skid).
Now Intel came up with something called Precise Event Based Sampling (PEBS) which stores a (partial) register set in some memory buffer at event time (trap like for some daft reason).
So from that we can obtain the IP of the instruction _after_ the instruction that caused the event. This is reliably so (mostly [*]) and does not contain out-of-order artifacts (0-skid).
So the ->precise flag tells us to use a more precise sampling method if available on the hardware (AMD could be using IBS to implement this for their instruction counter).
If you look at patch 9/14 you'll see we use the Last Branch Recording (LBR) facility of the Intel cpus (patch 8/14) to find the last basic block in the instruction stream and use that to rewind the instruction stream to get the actual instruction that triggered the event. In case that works I also set PERF_RECORD_MISC_EXACT to indicate we got the IP dead on (mostly [*]).
I suspect CPUs that are strictly in-order, like Atom, might always have it right, but I need to validate that.
Does that clarify stuff?
[*] there are CPU errata that may delay the PEBS recording, mostly with instructions like MOV SS, STI and things like SMM.
Re: [PATCH 06/14] perf, x86: PEBS infrastructure
Quoted 24 lines · Peter Zijlstra
Quoted 20 lines · Paul Mackerras
Quoted 7 lines · Peter Zijlstra
Implement a simple PEBS model that always takes a single PEBS event at a time. This is done so that the interaction with the rest of the system is as expected (freq adjust, period randomization, lbr).
Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
---...
Quoted 5 lines
(quoted diff) +0
-0@@ -203,8 +203,9 @@ struct perf_event_attr {
enable_on_exec: 1, /* next exec enables */ task : 1, /* trace fork/exit */ watermark : 1, /*wakeup_watermark*/
+ precise : 1, /* OoO invariant counter */Could you explain in a bit more detail what this means?
Quoted 2 lines
Also, it would be good to mention the ABI addition in the patch description, and explain it briefly there.
Quoted 8 lines
Quite so, my bad.
So on Intel regular PMIs can happen several instructions later than the actual event due to out-of-order processing of the instruction stream, that is, it doesn't keep the IP of the actual instruction that triggered the event, so all we have is the IP of where the interrupt happened (the difference between these IPs is called skid).
I don't think the skid is strictly related to OoO. You had skid on Itanium which was in-order.
Quoted 8 lines
Now Intel came up with something called Precise Event Based Sampling (PEBS) which stores a (partial) register set in some memory buffer at event time (trap like for some daft reason).
So from that we can obtain the IP of the instruction _after_ the instruction that caused the event. This is reliably so (mostly [*]) and does not contain out-of-order artifacts (0-skid).
The sampled instruction is guaranteed to have caused the event but it is unrelated to your sampling period, i.e., it can be N events after your sampling period due to the way PEBS is implemented.
Quoted 11 lines
So the ->precise flag tells us to use a more precise sampling method if available on the hardware (AMD could be using IBS to implement this for their instruction counter).
If you look at patch 9/14 you'll see we use the Last Branch Recording (LBR) facility of the Intel cpus (patch 8/14) to find the last basic block in the instruction stream and use that to rewind the instruction stream to get the actual instruction that triggered the event. In case that works I also set
PERF_RECORD_MISC_EXACTto indicate we got the IP dead on (mostly [*]).
By rewinding you get the IP of the instruction that caused the event. But the register state is STILL reflecting the situation at retirement of that instruction.
Quoted 3 lines
I suspect CPUs that are strictly in-order, like Atom, might always have it right, but I need to validate that.
Not sure about that.
Peter Zijlstra
maintainerRe: [PATCH 06/14] perf, x86: PEBS infrastructure
Quoted 3 lines · Stephane Eranian
By rewinding you get the IP of the instruction that caused the event. But the register state is STILL reflecting the situation at retirement of that instruction.
I'm quite aware of that, but its near impossible to reverse interpret instructions such as "mov rdx, $rdx", so until we get fault like PEBS we'll just have to live with this.
Re: [PATCH 06/14] perf, x86: PEBS infrastructure
Quoted 9 lines · Peter Zijlstra
Quoted 3 lines · Stephane Eranian
By rewinding you get the IP of the instruction that caused the event. But the register state is STILL reflecting the situation at retirement of that instruction.
I'm quite aware of that, but its near impossible to reverse interpret instructions such as "mov rdx, $rdx", so until we get fault like PEBS we'll just have to live with this.
I am sure you are, but that's not necessarily true of everybody. So this needs to be clearly spelled out in some documentation about precise + your MISC_EXACT bit.
[tip:perf/pebs] perf, x86: Add PEBS infrastructure
In reply to Peter ZijlstraCommit-ID: ca037701a025334e724e5c61b3b1082940c8b981 Gitweb: http://git.kernel.org/tip/ca037701a025334e724e5c61b3b1082940c8b981 Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 2 Mar 2010 19:52:12 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:31 +0100
perf, x86: Add PEBS infrastructure
This patch implements support for Intel Precise Event Based Sampling, which is an alternative counter mode in which the counter triggers a hardware assist to collect information on events. The hardware assist takes a trap like snapshot of a subset of the machine registers.
This data is written to the Intel Debug-Store, which can be programmed with a data threshold at which to raise a PMI.
With the PEBS hardware assist being trap like, the reported IP is always one instruction after the actual instruction that triggered the event.
This implements a simple PEBS model that always takes a single PEBS event at a time. This is done so that the interaction with the rest of the system is as expected (freq adjust, period randomization, lbr, callchains, etc.).
It adds an ABI element: perf_event_attr::precise, which indicates that we wish to use this (constrained, but precise) mode.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 223 ++++-------- arch/x86/kernel/cpu/perf_event_intel.c | 150 ++------ arch/x86/kernel/cpu/perf_event_intel_ds.c | 557 +++++++++++++++++++++++++++++ include/linux/perf_event.h | 3 +- 4 files changed, 671 insertions(+), 262 deletions(-)
arch/x86/kernel/cpu/perf_event.c +78 -145
@@ -31,45 +31,6 @@static u64 perf_event_mask__read_mostly ; -/* The maximal number of PEBS events: */ -#define MAX_PEBS_EVENTS 4 - -/* The size of a BTS record in bytes: */ -#define BTS_RECORD_SIZE 24 - -/* The size of a per-cpu BTS buffer in bytes: */ -#define BTS_BUFFER_SIZE (BTS_RECORD_SIZE * 2048) - -/* The BTS overflow threshold in bytes from the end of the buffer: */ -#define BTS_OVFL_TH (BTS_RECORD_SIZE * 128) - - -/* -* Bits in the debugctlmsr controlling branch tracing. -*/ -#define X86_DEBUGCTL_TR (1 << 6) -#define X86_DEBUGCTL_BTS (1 << 7) -#define X86_DEBUGCTL_BTINT (1 << 8) -#define X86_DEBUGCTL_BTS_OFF_OS (1 << 9) -#define X86_DEBUGCTL_BTS_OFF_USR (1 << 10) - -/* -* A debug store configuration. -* -* We only support architectures that use 64bit fields. -*/ -struct debug_store { -u64 bts_buffer_base; -u64 bts_index; -u64 bts_absolute_maximum; -u64 bts_interrupt_threshold; -u64 pebs_buffer_base; -u64 pebs_index; -u64 pebs_absolute_maximum; -u64 pebs_interrupt_threshold; -u64 pebs_event_reset[MAX_PEBS_EVENTS ]; -}; -struct event_constraint {union {unsigned long idxmsk[BITS_TO_LONGS (X86_PMC_IDX_MAX )];
@@ -88,17 +49,29 @@ struct amd_nb { };struct cpu_hw_events { +/* +* Generic x86 PMC bits +*/ struct perf_event *events[X86_PMC_IDX_MAX ];/* in counter order */ unsigned long active_mask[BITS_TO_LONGS (X86_PMC_IDX_MAX )];unsigned long interrupts;int enabled; -struct debug_store *ds;int n_events;int n_added;int assign[X86_PMC_IDX_MAX ];/* event to counter assignment */ u64 tags[X86_PMC_IDX_MAX ];struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ + +/* +* Intel DebugStore bits +*/ +struct debug_store *ds; +u64 pebs_enabled; + +/* +* AMD specific bits +*/ struct amd_nb *amd_nb; };
@@ -112,12 +85,24 @@ struct cpu_hw_events {#define EVENT_CONSTRAINT (c, n, m) \ __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n)) +/* +* Constraint on the Event code. +*/ #define INTEL_EVENT_CONSTRAINT (c, n) \ EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVTSEL_MASK) +/* +* Constraint on the Event code + UMask + fixed-mask +*/ #define FIXED_EVENT_CONSTRAINT (c, n) \ EVENT_CONSTRAINT(c, (1ULL << (32+n)), INTEL_ARCH_FIXED_MASK) +/* +* Constraint on the Event code + UMask +*/ +#define PEBS_EVENT_CONSTRAINT (c, n) \ + EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK) +#define EVENT_CONSTRAINT_END \ EVENT_CONSTRAINT(0, 0, 0)
@@ -128,6 +113,9 @@ struct cpu_hw_events {* struct x86_pmu - generic x86 pmu */ struct x86_pmu { +/* +* Generic x86 PMC bits +*/ const char *name;int version;int (*handle_irq)(struct pt_regs *);
@@ -146,10 +134,6 @@ struct x86_pmu {u64 event_mask;int apic;u64 max_period; -u64 intel_ctrl; -void (*enable_bts)(u64 config); -void (*disable_bts)(void ); -struct event_constraint * (*get_event_constraints)(struct cpu_hw_events *cpuc,struct perf_event *event);
@@ -162,6 +146,19 @@ struct x86_pmu {void (*cpu_starting)(int cpu);void (*cpu_dying)(int cpu);void (*cpu_dead)(int cpu); + +/* +* Intel Arch Perfmon v2+ +*/ +u64 intel_ctrl; + +/* +* Intel DebugStore bits +*/ +int bts, pebs; +int pebs_record_size; +void (*drain_pebs)(struct pt_regs *regs); +struct event_constraint *pebs_constraints; };static struct x86_pmu x86_pmu__read_mostly ;
@@ -293,110 +290,14 @@ static void release_pmc_hardware(void)#endif } -static inline bool bts_available (void ) -{ -return x86_pmu.enable_bts !=NULL ; -} - -static void init_debug_store_on_cpu (int cpu) -{ -struct debug_store *ds =per_cpu (cpu_hw_events, cpu).ds; - -if (!ds) -return ; - -wrmsr_on_cpu (cpu,MSR_IA32_DS_AREA , - (u32)((u64 )(unsigned long )ds), - (u32)((u64 )(unsigned long )ds >>32 )); -} - -static void fini_debug_store_on_cpu (int cpu) -{ -if (!per_cpu (cpu_hw_events, cpu).ds) -return ; - -wrmsr_on_cpu (cpu,MSR_IA32_DS_AREA ,0 ,0 ); -} - -static void release_bts_hardware (void ) -{ -int cpu; - -if (!bts_available ()) -return ; - -get_online_cpus (); - -for_each_online_cpu (cpu) -fini_debug_store_on_cpu (cpu); - -for_each_possible_cpu (cpu) { -struct debug_store *ds =per_cpu (cpu_hw_events, cpu).ds; - -if (!ds) -continue ; - -per_cpu (cpu_hw_events, cpu).ds =NULL ; - -kfree ((void *)(unsigned long )ds->bts_buffer_base); -kfree (ds); - } - -put_online_cpus (); -} - -static int reserve_bts_hardware (void ) -{ -int cpu, err =0 ; - -if (!bts_available ()) -return 0 ; - -get_online_cpus (); - -for_each_possible_cpu (cpu) { -struct debug_store *ds; -void *buffer; - - err = -ENOMEM ; - buffer =kzalloc (BTS_BUFFER_SIZE ,GFP_KERNEL ); -if (unlikely (!buffer)) -break ; - - ds =kzalloc (sizeof (*ds),GFP_KERNEL ); -if (unlikely (!ds)) { -kfree (buffer); -break ; - } - - ds->bts_buffer_base = (u64 )(unsigned long )buffer; - ds->bts_index = ds->bts_buffer_base; - ds->bts_absolute_maximum = - ds->bts_buffer_base +BTS_BUFFER_SIZE ; - ds->bts_interrupt_threshold = - ds->bts_absolute_maximum -BTS_OVFL_TH ; - -per_cpu (cpu_hw_events, cpu).ds = ds; - err =0 ; - } - -if (err) -release_bts_hardware (); -else { -for_each_online_cpu (cpu) -init_debug_store_on_cpu (cpu); - } - -put_online_cpus (); - -return err; -} +static int reserve_ds_buffers (void ); +static void release_ds_buffers (void );static void hw_perf_event_destroy (struct perf_event *event) {if (atomic_dec_and_mutex_lock (&active_events, &pmc_reserve_mutex)) {release_pmc_hardware (); -release_bts_hardware (); +release_ds_buffers ();mutex_unlock (&pmc_reserve_mutex); } }
@@ -459,7 +360,7 @@ static int __hw_perf_event_init(struct perf_event *event)if (!reserve_pmc_hardware ()) err = -EBUSY ;else - err =reserve_bts_hardware (); + err =reserve_ds_buffers (); }if (!err)atomic_inc (&active_events);
@@ -537,7 +438,7 @@ static int __hw_perf_event_init(struct perf_event *event)if ((attr->config ==PERF_COUNT_HW_BRANCH_INSTRUCTIONS ) && (hwc->sample_period ==1 )) {/* BTS is not supported by this architecture. */ -if (!bts_available ()) +if (!x86_pmu.bts)return -EOPNOTSUPP ;/* BTS is currently only allowed for user-mode. */
@@ -995,6 +896,7 @@ static void x86_pmu_unthrottle(struct perf_event *event)void perf_event_print_debug (void ) {u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; +u64 pebs;struct cpu_hw_events *cpuc;unsigned long flags;int cpu, idx;
@@ -1012,12 +914,14 @@ void perf_event_print_debug(void)rdmsrl (MSR_CORE_PERF_GLOBAL_STATUS , status);rdmsrl (MSR_CORE_PERF_GLOBAL_OVF_CTRL , overflow);rdmsrl (MSR_ARCH_PERFMON_FIXED_CTR_CTRL , fixed); +rdmsrl (MSR_IA32_PEBS_ENABLE , pebs);pr_info ("\n" );pr_info ("CPU#%d: ctrl: %016llx\n" , cpu, ctrl);pr_info ("CPU#%d: status: %016llx\n" , cpu, status);pr_info ("CPU#%d: overflow: %016llx\n" , cpu, overflow);pr_info ("CPU#%d: fixed: %016llx\n" , cpu, fixed); +pr_info ("CPU#%d: pebs: %016llx\n" , cpu, pebs); }pr_info ("CPU#%d: active: %016llx\n" , cpu, *(u64 *)cpuc->active_mask);
@@ -1333,6 +1237,7 @@ undo:#include "perf_event_amd.c" #include "perf_event_p6.c" +#include "perf_event_intel_ds.c" #include "perf_event_intel.c" static int __cpuinit
@@ -1465,6 +1370,32 @@ static const struct pmu pmu = { };/* +* validate that we can schedule this event +*/ +static int validate_event (struct perf_event *event) +{ +struct cpu_hw_events *fake_cpuc; +struct event_constraint *c; +int ret =0 ; + + fake_cpuc =kmalloc (sizeof (*fake_cpuc),GFP_KERNEL | __GFP_ZERO); +if (!fake_cpuc) +return -ENOMEM ; + + c = x86_pmu.get_event_constraints (fake_cpuc, event); + +if (!c || !c->weight) + ret = -ENOSPC ; + +if (x86_pmu.put_event_constraints) + x86_pmu.put_event_constraints (fake_cpuc, event); + +kfree (fake_cpuc); + +return ret; +} + +/* * validate a single event group * * validation include:
@@ -1529,6 +1460,8 @@ const struct pmu *hw_perf_event_init(struct perf_event *event)if (event->group_leader != event) err =validate_group (event); +else + err =validate_event (event); event->pmu = tmp; }
arch/x86/kernel/cpu/perf_event_intel.c +34 -116
229 lines not on this page · raw message
arch/x86/kernel/cpu/perf_event_intel_ds.c new file +557 -0
557 lines not on this page · raw message
include/linux/perf_event.h +2 -1
@@ -203,8 +203,9 @@ struct perf_event_attr { enable_on_exec : 1, /* next exec enables */ task : 1, /* trace fork/exit */ watermark : 1, /* wakeup_watermark */ + precise : 1, /* OoO invariant counter */ - __reserved_1 : 49; + __reserved_1 : 48;union {__u32 wakeup_events;/* wakeup every n events */
PATCH 7/14 perf: Add attr->precise support to raw event parsing
In reply to Peter ZijlstraMinimal userspace interface to the new 'precise' events flag.
Can be used like: perf top -e r00c0p
Which will use PEBS to sample retired instructions
LKML-Reference: <new-submission>
--- tools/perf/util/parse-events.c | 5 +++++ 1 file changed, 5 insertions(+)
tools/perf/util/parse-events.c +4 -0
@@ -656,6 +656,10 @@ parse_raw_event(const char **strp, strucreturn EVT_FAILED ; n =hex2u64 (str +1 , &config);if (n >0 ) { +if (str[n+1 ] =='p' ) { + attr->precise =1 ; + n++; + } *strp = str + n +1 ; attr->type =PERF_TYPE_RAW ; attr->config = config;
[tip:perf/pebs] perf: Add attr->precise support to raw event parsing
Commit-ID: 69fef0d2e2c2c049ef4207a52e78b50d527bd85a Gitweb: http://git.kernel.org/tip/69fef0d2e2c2c049ef4207a52e78b50d527bd85a Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 4 Mar 2010 13:57:24 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:32 +0100
perf: Add attr->precise support to raw event parsing
Minimal userspace interface to the new 'precise' events flag.
Can be used like "perf top -e r00c0p" which will use PEBS to sample retired instructions.
LKML-Reference: <[email protected]>
--- tools/perf/util/parse-events.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
tools/perf/util/parse-events.c +4 -0
@@ -656,6 +656,10 @@ parse_raw_event(const char **strp, struct perf_event_attr *attr)return EVT_FAILED ; n =hex2u64 (str +1 , &config);if (n >0 ) { +if (str[n+1 ] =='p' ) { + attr->precise =1 ; + n++; + } *strp = str + n +1 ; attr->type =PERF_TYPE_RAW ; attr->config = config;
PATCH 8/14 perf, x86: Implement simple LBR support
In reply to Peter ZijlstraImplement support for Intel LBR stacks that support FREEZE_LBRS_ON_PMI. We do not (yet?) support the LBR config register because that is SMT wide and would also put undue restraints on the PEBS users.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 18 ++ arch/x86/kernel/cpu/perf_event_intel.c | 13 + arch/x86/kernel/cpu/perf_event_intel_lbr.c | 228 +++++++++++++++++++++++++++++ include/linux/perf_event.h | 11 + 4 files changed, 270 insertions(+)
arch/x86/kernel/cpu/perf_event.c +18 -0
@@ -48,6 +48,8 @@ struct amd_nb {struct event_constraint event_constraints[X86_PMC_IDX_MAX ]; }; +#define MAX_LBR_ENTRIES 16 +struct cpu_hw_events {/* * Generic x86 PMC bits
@@ -70,6 +72,14 @@ struct cpu_hw_events {u64 pebs_enabled;/* +* Intel LBR bits +*/ +int lbr_users; +void *lbr_context; +struct perf_branch_stack lbr_stack; +struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES ]; + +/* * AMD specific bits */ struct amd_nb *amd_nb;
@@ -154,6 +164,13 @@ struct x86_pmu {int pebs_record_size;void (*drain_pebs)(struct pt_regs *regs);struct event_constraint *pebs_constraints; + +/* +* Intel LBR +*/ +unsigned long lbr_tos, lbr_from, lbr_to;/* MSR base regs */ +int lbr_nr;/* hardware stack size */ +int lbr_format;/* hardware format */ };static struct x86_pmu x86_pmu__read_mostly ;
@@ -1238,6 +1255,7 @@ undo:#include "perf_event_amd.c" #include "perf_event_p6.c" +#include "perf_event_intel_lbr.c" #include "perf_event_intel_ds.c" #include "perf_event_intel.c"
arch/x86/kernel/cpu/perf_event_intel.c +13 -0
@@ -480,6 +480,7 @@ static void intel_pmu_disable_all(void)intel_pmu_disable_bts ();intel_pmu_pebs_disable_all (); +intel_pmu_lbr_disable_all (); }static void intel_pmu_enable_all (void )
@@ -499,6 +500,7 @@ static void intel_pmu_enable_all(void) }intel_pmu_pebs_enable_all (); +intel_pmu_lbr_enable_all (); }static inline u64 intel_pmu_get_status (void )
@@ -675,6 +677,8 @@ again:inc_irq_stat (apic_perf_irqs); ack = status; +intel_pmu_lbr_read (); +/* * PEBS overflow sets bit 62 in the global status register */
@@ -847,6 +851,8 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, core2_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_core (); + x86_pmu.event_constraints = intel_core2_event_constraints;pr_cont ("Core2 events, " );break ;
@@ -856,13 +862,18 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, nehalem_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_nhm (); + x86_pmu.event_constraints = intel_nehalem_event_constraints;pr_cont ("Nehalem/Corei7 events, " );break ; +case 28 :/* Atom */ memcpy (hw_cache_event_ids, atom_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_atom (); + x86_pmu.event_constraints = intel_gen_event_constraints;pr_cont ("Atom events, " );break ;
@@ -872,6 +883,8 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, westmere_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_nhm (); + x86_pmu.event_constraints = intel_westmere_event_constraints;pr_cont ("Westmere events, " );break ;
arch/x86/kernel/cpu/perf_event_intel_lbr.c new file +228 -0
228 lines not on this page · raw message
include/linux/perf_event.h +11 -0
@@ -467,6 +467,17 @@ struct perf_raw_record {void *data; }; +struct perf_branch_entry { +__u64 from; +__u64 to; +__u64 flags; +}; + +struct perf_branch_stack { +__u64 nr; +struct perf_branch_entry entries[0 ]; +}; +struct task_struct ;/**
[tip:perf/pebs] perf, x86: Implement simple LBR support
Commit-ID: caff2befffe899e63df5cc760b7ed01cfd902685 Gitweb: http://git.kernel.org/tip/caff2befffe899e63df5cc760b7ed01cfd902685 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 3 Mar 2010 12:02:30 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:32 +0100
perf, x86: Implement simple LBR support
Implement simple suport Intel Last-Branch-Record, it supports all hardware that implements FREEZE_LBRS_ON_PMI, but does not (yet) implement the LBR config register.
The Intel LBR is a FIFO of From,To addresses describing the last few branches the hardware took.
This patch does not add perf interface to the LBR, but merely provides an interface for internal use.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 18 +++ arch/x86/kernel/cpu/perf_event_intel.c | 13 ++ arch/x86/kernel/cpu/perf_event_intel_lbr.c | 228 ++++++++++++++++++++++++++++ include/linux/perf_event.h | 11 ++ 4 files changed, 270 insertions(+), 0 deletions(-)
arch/x86/kernel/cpu/perf_event.c +18 -0
@@ -48,6 +48,8 @@ struct amd_nb {struct event_constraint event_constraints[X86_PMC_IDX_MAX ]; }; +#define MAX_LBR_ENTRIES 16 +struct cpu_hw_events {/* * Generic x86 PMC bits
@@ -70,6 +72,14 @@ struct cpu_hw_events {u64 pebs_enabled;/* +* Intel LBR bits +*/ +int lbr_users; +void *lbr_context; +struct perf_branch_stack lbr_stack; +struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES ]; + +/* * AMD specific bits */ struct amd_nb *amd_nb;
@@ -159,6 +169,13 @@ struct x86_pmu {int pebs_record_size;void (*drain_pebs)(struct pt_regs *regs);struct event_constraint *pebs_constraints; + +/* +* Intel LBR +*/ +unsigned long lbr_tos, lbr_from, lbr_to;/* MSR base regs */ +int lbr_nr;/* hardware stack size */ +int lbr_format;/* hardware format */ };static struct x86_pmu x86_pmu__read_mostly ;
@@ -1237,6 +1254,7 @@ undo:#include "perf_event_amd.c" #include "perf_event_p6.c" +#include "perf_event_intel_lbr.c" #include "perf_event_intel_ds.c" #include "perf_event_intel.c"
arch/x86/kernel/cpu/perf_event_intel.c +13 -0
@@ -480,6 +480,7 @@ static void intel_pmu_disable_all(void)intel_pmu_disable_bts ();intel_pmu_pebs_disable_all (); +intel_pmu_lbr_disable_all (); }static void intel_pmu_enable_all (void )
@@ -499,6 +500,7 @@ static void intel_pmu_enable_all(void) }intel_pmu_pebs_enable_all (); +intel_pmu_lbr_enable_all (); }static inline u64 intel_pmu_get_status (void )
@@ -674,6 +676,8 @@ again:inc_irq_stat (apic_perf_irqs); ack = status; +intel_pmu_lbr_read (); +/* * PEBS overflow sets bit 62 in the global status register */
@@ -848,6 +852,8 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, core2_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_core (); + x86_pmu.event_constraints = intel_core2_event_constraints;pr_cont ("Core2 events, " );break ;
@@ -857,13 +863,18 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, nehalem_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_nhm (); + x86_pmu.event_constraints = intel_nehalem_event_constraints;pr_cont ("Nehalem/Corei7 events, " );break ; +case 28 :/* Atom */ memcpy (hw_cache_event_ids, atom_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_atom (); + x86_pmu.event_constraints = intel_gen_event_constraints;pr_cont ("Atom events, " );break ;
@@ -873,6 +884,8 @@ static __init int intel_pmu_init(void)memcpy (hw_cache_event_ids, westmere_hw_cache_event_ids,sizeof (hw_cache_event_ids)); +intel_pmu_lbr_init_nhm (); + x86_pmu.event_constraints = intel_westmere_event_constraints;pr_cont ("Westmere events, " );break ;
arch/x86/kernel/cpu/perf_event_intel_lbr.c new file +228 -0
228 lines not on this page · raw message
include/linux/perf_event.h +11 -0
@@ -467,6 +467,17 @@ struct perf_raw_record {void *data; }; +struct perf_branch_entry { +__u64 from; +__u64 to; +__u64 flags; +}; + +struct perf_branch_stack { +__u64 nr; +struct perf_branch_entry entries[0 ]; +}; +struct task_struct ;/**
PATCH 9/14 perf, x86: use LBR for PEBS IP+1 fixup
In reply to Peter ZijlstraPEBS always reports the IP+1, that is the instruction after the one that got sampled, cure this by using the LBR to reliably rewind the instruction stream.
LKML-Reference: <new-submission>
--- arch/x86/include/asm/perf_event.h | 19 ++++++ arch/x86/kernel/cpu/perf_event.c | 70 ++++++++++++------------- arch/x86/kernel/cpu/perf_event_intel.c | 4 - arch/x86/kernel/cpu/perf_event_intel_ds.c | 84 +++++++++++++++++++++++++++++- include/linux/perf_event.h | 6 ++ 5 files changed, 144 insertions(+), 39 deletions(-)
arch/x86/kernel/cpu/perf_event.c +35 -35
@@ -29,6 +29,41 @@#include <asm/stacktrace.h> #include <asm/nmi.h> +/* +* best effort, GUP based copy_from_user() that assumes IRQ or NMI context +*/ +static unsigned long +copy_from_user_nmi (void *to,const void __user *from,unsigned long n) +{ +unsigned long offset, addr = (unsigned long )from; +int type =in_nmi () ?KM_NMI :KM_IRQ0 ; +unsigned long size, len =0 ; +struct page *page; +void *map; +int ret; + +do { + ret =__get_user_pages_fast (addr,1 ,0 , &page); +if (!ret) +break ; + + offset = addr & (PAGE_SIZE -1 ); + size =min (PAGE_SIZE - offset, n - len); + + map =kmap_atomic (page, type); +memcpy (to, map+offset, size); +kunmap_atomic (map, type); +put_page (page); + + len += size; + to += size; + addr += size; + + }while (len < n); + +return len; +} +static u64 perf_event_mask__read_mostly ;struct event_constraint {
@@ -1516,41 +1551,6 @@ perf_callchain_kernel(struct pt_regs *redump_trace (NULL , regs,NULL , regs->bp, &backtrace_ops, entry); } -/* -* best effort, GUP based copy_from_user() that assumes IRQ or NMI context -*/ -static unsigned long -copy_from_user_nmi (void *to,const void __user *from,unsigned long n) -{ -unsigned long offset, addr = (unsigned long )from; -int type =in_nmi () ?KM_NMI :KM_IRQ0 ; -unsigned long size, len =0 ; -struct page *page; -void *map; -int ret; - -do { - ret =__get_user_pages_fast (addr,1 ,0 , &page); -if (!ret) -break ; - - offset = addr & (PAGE_SIZE -1 ); - size =min (PAGE_SIZE - offset, n - len); - - map =kmap_atomic (page, type); -memcpy (to, map+offset, size); -kunmap_atomic (map, type); -put_page (page); - - len += size; - to += size; - addr += size; - - }while (len < n); - -return len; -} -static int copy_stack_frame (const void __user *fp,struct stack_frame *frame) {unsigned long bytes;
arch/x86/kernel/cpu/perf_event_intel.c +2 -2
@@ -547,7 +547,7 @@ static void intel_pmu_disable_event(strux86_pmu_disable_event (event);if (unlikely (event->attr.precise)) -intel_pmu_pebs_disable (hwc); +intel_pmu_pebs_disable (event); }static void intel_pmu_enable_fixed (struct hw_perf_event *hwc)
@@ -600,7 +600,7 @@ static void intel_pmu_enable_event(struc }if (unlikely (event->attr.precise)) -intel_pmu_pebs_enable (hwc); +intel_pmu_pebs_enable (event);__x86_pmu_enable_event (hwc); }
arch/x86/kernel/cpu/perf_event_intel_ds.c +82 -2
126 lines not on this page · raw message
arch/x86/include/asm/perf_event.h +19 -0
@@ -136,6 +136,25 @@ extern void perf_events_lapic_init(void)#define PERF_EVENT_INDEX_OFFSET 0 +/* +* Abuse bit 3 of the cpu eflags register to indicate proper PEBS IP fixups. +* This flag is otherwise unused and ABI specified to be 0, so nobody should +* care what we do with it. +*/ +#define PERF_EFLAGS_EXACT (1UL << 3) + +#define perf_misc_flags (regs) \ +({ int misc = 0; \ + if (user_mode(regs)) \ + misc |= PERF_RECORD_MISC_USER; \ + else \ + misc |= PERF_RECORD_MISC_KERNEL; \ + if (regs->flags & PERF_EFLAGS_EXACT) \ + misc |= PERF_RECORD_MISC_EXACT; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip) +#else static inline void init_hw_perf_events (void ) { }static inline void perf_events_lapic_init (void ) { }
include/linux/perf_event.h +6 -0
@@ -294,6 +294,12 @@ struct perf_event_mmap_page {#define PERF_RECORD_MISC_USER (2 << 0)#define PERF_RECORD_MISC_HYPERVISOR (3 << 0) +#define PERF_RECORD_MISC_EXACT (1 << 14) +/* +* Reserve the last bit to indicate some extended misc field +*/ +#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15) +struct perf_event_header {__u32 type;__u16 misc;
Re: [PATCH 09/14] perf, x86: use LBR for PEBS IP+1 fixup
Quoted 16 lines · Peter Zijlstra
PEBS always reports the IP+1, that is the instruction after the one that got sampled, cure this by using the LBR to reliably rewind the instruction stream.
CC Masami Hiramatsu <[email protected]>CC Yanmin Zhang <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
--- arch/x86/include/asm/perf_event.h | 19 ++++++ arch/x86/kernel/cpu/perf_event.c | 70 ++++++++++++------------- arch/x86/kernel/cpu/perf_event_intel.c | 4 - arch/x86/kernel/cpu/perf_event_intel_ds.c | 84 +++++++++++++++++++++++++++++- include/linux/perf_event.h | 6 ++ 5 files changed, 144 insertions(+), 39 deletions(-)
[...]
Quoted 16 lines
Index:
linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================arch/x86/include/asm/perf_event.h +8
-0@@ -136,6 +136,25 @@ extern void perf_events_lapic_init(void)#define PERF_EVENT_INDEX_OFFSET 0 +/* +* Abuse bit 3 of the cpu eflags register to indicate proper PEBS IP fixups. +* This flag is otherwise unused and ABI specified to be 0, so nobody should +* care what we do with it. +*/ +#define PERF_EFLAGS_EXACT (1UL << 3) + +#define perf_misc_flags (regs) \
arch/x86/include/asm/perf_event.h extern void perf_events_lapic_init(void) · line 156
Quoted 10 lines
arch/x86/include/asm/perf_event.h +10
-0+({int misc =0 ; \ +if (user_mode (regs)) \ + misc |=PERF_RECORD_MISC_USER ; \ +else \ + misc |=PERF_RECORD_MISC_KERNEL ; \ +if (regs->flags &PERF_EFLAGS_EXACT ) \ + misc |=PERF_RECORD_MISC_EXACT ; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip)
Hmm, why don't you use instruction_pointer() defined in asm/ptrace.h? And I couldn't find any user of this macro in this patch...
Others looks good to me :)
Thank you,
Signature · 2 lines
Masami Hiramatsu e-mail: [email protected]
Peter Zijlstra
maintainerRe: [PATCH 09/14] perf, x86: use LBR for PEBS IP+1 fixup
Quoted 16 lines · Masami Hiramatsu
arch/x86/include/asm/perf_event.h extern void perf_events_lapic_init(void) · line 156
Quoted 11 lines · Peter Zijlstra
arch/x86/include/asm/perf_event.h +11
-0+#define perf_misc_flags (regs) \ +({ int misc = 0; \ + if (user_mode(regs)) \ + misc |= PERF_RECORD_MISC_USER; \ + else \ + misc |= PERF_RECORD_MISC_KERNEL; \ + if (regs->flags & PERF_EFLAGS_EXACT) \ + misc |= PERF_RECORD_MISC_EXACT; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip)Hmm, why don't you use
instruction_pointer()defined inasm/ptrace.h? And I couldn't find any user of this macro in this patch...
perf_instruction_pointer() is used in kernel/perf_event.c, and yeah I could have used instruction_pointer() but that's yet another wrapper.
Anyway, Yanmin is poking at doing kvm-guest profiling and will likely rewrite all of the perf_misc() and perf_instruction_pointer() stuff soon.
Re: [PATCH 09/14] perf, x86: use LBR for PEBS IP+1 fixup
Quoted 18 lines · Peter Zijlstra
Quoted 16 lines · Masami Hiramatsu
arch/x86/include/asm/perf_event.h extern void perf_events_lapic_init(void) · line 156
Quoted 11 lines · Peter Zijlstra
arch/x86/include/asm/perf_event.h +11
-0+#define perf_misc_flags (regs) \ +({ int misc = 0; \ + if (user_mode(regs)) \ + misc |= PERF_RECORD_MISC_USER; \ + else \ + misc |= PERF_RECORD_MISC_KERNEL; \ + if (regs->flags & PERF_EFLAGS_EXACT) \ + misc |= PERF_RECORD_MISC_EXACT; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip)Hmm, why don't you use
instruction_pointer()defined inasm/ptrace.h? And I couldn't find any user of this macro in this patch...
Quoted 6 lines
perf_instruction_pointer()is used inkernel/perf_event.c, and yeah I could have usedinstruction_pointer()but that's yet another wrapper.Anyway, Yanmin is poking at doing kvm-guest profiling and will likely rewrite all of the
perf_misc()andperf_instruction_pointer()stuff soon.
Hmm, still I can't find where it is used (in your patches). Anyway, would you mean that perf_instruction_pointer() will have different implementation by Yanmin?
Thank you,
Signature · 2 lines
Masami Hiramatsu e-mail: [email protected]
Masami Hiramatsu
Reviewed-byRe: [PATCH 09/14] perf, x86: use LBR for PEBS IP+1 fixup
Quoted 19 lines · Masami Hiramatsu
Quoted 18 lines · Peter Zijlstra
Quoted 16 lines · Masami Hiramatsu
arch/x86/include/asm/perf_event.h extern void perf_events_lapic_init(void) · line 156
Quoted 11 lines · Peter Zijlstra
arch/x86/include/asm/perf_event.h +11
-0+#define perf_misc_flags (regs) \ +({ int misc = 0; \ + if (user_mode(regs)) \ + misc |= PERF_RECORD_MISC_USER; \ + else \ + misc |= PERF_RECORD_MISC_KERNEL; \ + if (regs->flags & PERF_EFLAGS_EXACT) \ + misc |= PERF_RECORD_MISC_EXACT; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip)Hmm, why don't you use
instruction_pointer()defined inasm/ptrace.h? And I couldn't find any user of this macro in this patch...
Quoted 8 lines
Quoted 6 lines
perf_instruction_pointer()is used inkernel/perf_event.c, and yeah I could have usedinstruction_pointer()but that's yet another wrapper.Anyway, Yanmin is poking at doing kvm-guest profiling and will likely rewrite all of the
perf_misc()andperf_instruction_pointer()stuff soon.Hmm, still I can't find where it is used (in your patches).
Ah, OK. that is already used in kernel/perf_event.c and the patch overwrites one defined in include/linux/perf_event.h.
Thank you,
Signature · 2 lines
Masami Hiramatsu e-mail: [email protected]
Peter Zijlstra
maintainerRe: [PATCH 09/14] perf, x86: use LBR for PEBS IP+1 fixup
In reply to Masami HiramatsuQuoted 2 lines · Masami Hiramatsu
Anyway, would you mean that
perf_instruction_pointer()will have different implementation by Yanmin?
Yes, he will also be checking if we came from guest context:
[tip:perf/pebs] perf, x86: use LBR for PEBS IP+1 fixup
In reply to Peter ZijlstraCommit-ID: ef21f683a045a79b6aa86ad81e5fdfc0d5ddd250 Gitweb: http://git.kernel.org/tip/ef21f683a045a79b6aa86ad81e5fdfc0d5ddd250 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 3 Mar 2010 13:12:23 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:32 +0100
perf, x86: use LBR for PEBS IP+1 fixup
Use the LBR to fix up the PEBS IP+1 issue.
As said, PEBS reports the next instruction, here we use the LBR to find the last branch and from that construct the actual IP. If the IP matches the LBR-TO, we use LBR-FROM, otherwise we use the LBR-TO address as the beginning of the last basic block and decode forward.
Once we find a match to the current IP, we use the previous location.
This patch introduces a new ABI element: PERF_RECORD_MISC_EXACT, which conveys that the reported IP (PERF_SAMPLE_IP) is the exact instruction that caused the event (barring CPU errata).
The fixup can fail due to various reasons:
1) LBR contains invalid data (quite possible)
2) part of the basic block got paged out
3) the reported IP isn't part of the basic block (see 1)
LKML-Reference: <[email protected]>
--- arch/x86/include/asm/perf_event.h | 19 +++++++ arch/x86/kernel/cpu/perf_event.c | 70 ++++++++++++------------ arch/x86/kernel/cpu/perf_event_intel.c | 4 +- arch/x86/kernel/cpu/perf_event_intel_ds.c | 84 ++++++++++++++++++++++++++++- include/linux/perf_event.h | 6 ++ 5 files changed, 144 insertions(+), 39 deletions(-)
arch/x86/include/asm/perf_event.h +19 -0
@@ -136,6 +136,25 @@ extern void perf_events_lapic_init(void);#define PERF_EVENT_INDEX_OFFSET 0 +/* +* Abuse bit 3 of the cpu eflags register to indicate proper PEBS IP fixups. +* This flag is otherwise unused and ABI specified to be 0, so nobody should +* care what we do with it. +*/ +#define PERF_EFLAGS_EXACT (1UL << 3) + +#define perf_misc_flags (regs) \ +({ int misc = 0; \ + if (user_mode(regs)) \ + misc |= PERF_RECORD_MISC_USER; \ + else \ + misc |= PERF_RECORD_MISC_KERNEL; \ + if (regs->flags & PERF_EFLAGS_EXACT) \ + misc |= PERF_RECORD_MISC_EXACT; \ + misc; }) + +#define perf_instruction_pointer (regs) ((regs)->ip) +#else static inline void init_hw_perf_events (void ) { }static inline void perf_events_lapic_init (void ) { }
arch/x86/kernel/cpu/perf_event.c +35 -35
82 lines not on this page · raw message
arch/x86/kernel/cpu/perf_event_intel.c +2 -2
@@ -547,7 +547,7 @@ static void intel_pmu_disable_event(struct perf_event *event)x86_pmu_disable_event (event);if (unlikely (event->attr.precise)) -intel_pmu_pebs_disable (hwc); +intel_pmu_pebs_disable (event); }static void intel_pmu_enable_fixed (struct hw_perf_event *hwc)
@@ -600,7 +600,7 @@ static void intel_pmu_enable_event(struct perf_event *event) }if (unlikely (event->attr.precise)) -intel_pmu_pebs_enable (hwc); +intel_pmu_pebs_enable (event);__x86_pmu_enable_event (hwc); }
arch/x86/kernel/cpu/perf_event_intel_ds.c +82 -2
126 lines not on this page · raw message
include/linux/perf_event.h +6 -0
@@ -294,6 +294,12 @@ struct perf_event_mmap_page {#define PERF_RECORD_MISC_USER (2 << 0)#define PERF_RECORD_MISC_HYPERVISOR (3 << 0) +#define PERF_RECORD_MISC_EXACT (1 << 14) +/* +* Reserve the last bit to indicate some extended misc field +*/ +#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15) +struct perf_event_header {__u32 type;__u16 misc;
PATCH 10/14 perf-top: Show the percentage of successfull PEBS-fixups
In reply to Peter ZijlstraUse the PERF_RECORD_MISC_EXACT information to measure the success rate of the PEBS fix-up.
LKML-Reference: <new-submission>
--- tools/perf/builtin-top.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
tools/perf/builtin-top.c +9 -3
@@ -410,6 +410,7 @@ static double sym_weight(const struct systatic long samples;static long userspace_samples; +static long exact_samples;static const char CONSOLE_CLEAR [] ="[H[2J" ;static void __list_insert_active_sym (struct sym_entry *syme)
@@ -450,6 +451,7 @@ static void print_sym_table(void)int counter, snap = !display_weighted ? sym_counter :0 ;float samples_per_sec = samples/delay_secs;float ksamples_per_sec = (samples-userspace_samples)/delay_secs; +float esamples_percent = (100.0 *exact_samples)/samples;float sum_ksamples =0.0 ;struct sym_entry *syme, *n;struct rb_root tmp =RB_ROOT ;
@@ -457,7 +459,7 @@ static void print_sym_table(void)int sym_width =0 , dso_width =0 , max_dso_width;const int win_width = winsize.ws_col -1 ; - samples = userspace_samples =0 ; + samples = userspace_samples = exact_samples =0 ;/* Sort the active symbols */ pthread_mutex_lock (&active_symbols_lock);
@@ -488,9 +490,10 @@ static void print_sym_table(void)puts (CONSOLE_CLEAR );printf ("%-*.*s\n" , win_width, win_width, graph_dotted_line); -printf (" PerfTop:%8.0f irqs/sec kernel:%4.1f%% [" , +printf (" PerfTop:%8.0f irqs/sec kernel:%4.1f%% exact: %4.1f%% [" , samples_per_sec, -100.0 - (100.0 *((samples_per_sec-ksamples_per_sec)/samples_per_sec))); +100.0 - (100.0 *((samples_per_sec-ksamples_per_sec)/samples_per_sec)), + esamples_percent);if (nr_counters ==1 || !display_weighted) {printf ("%Ld" , (u64 )attrs[0 ].sample_period);
@@ -954,6 +957,9 @@ static void event__process_sample(constreturn ; } +if (self->header.misc &PERF_RECORD_MISC_EXACT ) + exact_samples++; +if (event__preprocess_sample (self, session, &al, symbol_filter) <0 || al.filtered)return ;
[tip:perf/pebs] perf-top: Show the percentage of successfull PEBS-fixups
Commit-ID: 1676b8a077c352085d52578fb4f29350b58b6e74 Gitweb: http://git.kernel.org/tip/1676b8a077c352085d52578fb4f29350b58b6e74 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 4 Mar 2010 14:19:36 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:33 +0100
perf-top: Show the percentage of successfull PEBS-fixups
Use the PERF_RECORD_MISC_EXACT information to measure the success rate of the PEBS fix-up.
LKML-Reference: <[email protected]>
--- tools/perf/builtin-top.c | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-)
tools/perf/builtin-top.c +9 -3
@@ -410,6 +410,7 @@ static double sym_weight(const struct sym_entry *sym)static long samples;static long userspace_samples; +static long exact_samples;static const char CONSOLE_CLEAR [] ="[H[2J" ;static void __list_insert_active_sym (struct sym_entry *syme)
@@ -450,6 +451,7 @@ static void print_sym_table(void)int counter, snap = !display_weighted ? sym_counter :0 ;float samples_per_sec = samples/delay_secs;float ksamples_per_sec = (samples-userspace_samples)/delay_secs; +float esamples_percent = (100.0 *exact_samples)/samples;float sum_ksamples =0.0 ;struct sym_entry *syme, *n;struct rb_root tmp =RB_ROOT ;
@@ -457,7 +459,7 @@ static void print_sym_table(void)int sym_width =0 , dso_width =0 , max_dso_width;const int win_width = winsize.ws_col -1 ; - samples = userspace_samples =0 ; + samples = userspace_samples = exact_samples =0 ;/* Sort the active symbols */ pthread_mutex_lock (&active_symbols_lock);
@@ -488,9 +490,10 @@ static void print_sym_table(void)puts (CONSOLE_CLEAR );printf ("%-*.*s\n" , win_width, win_width, graph_dotted_line); -printf (" PerfTop:%8.0f irqs/sec kernel:%4.1f%% [" , +printf (" PerfTop:%8.0f irqs/sec kernel:%4.1f%% exact: %4.1f%% [" , samples_per_sec, -100.0 - (100.0 *((samples_per_sec-ksamples_per_sec)/samples_per_sec))); +100.0 - (100.0 *((samples_per_sec-ksamples_per_sec)/samples_per_sec)), + esamples_percent);if (nr_counters ==1 || !display_weighted) {printf ("%Ld" , (u64 )attrs[0 ].sample_period);
@@ -954,6 +957,9 @@ static void event__process_sample(const event_t *self,return ; } +if (self->header.misc &PERF_RECORD_MISC_EXACT ) + exact_samples++; +if (event__preprocess_sample (self, session, &al, symbol_filter) <0 || al.filtered)return ;
PATCH 11/14 perf, x86: Clean up IA32_PERF_CAPABILITIES usage
In reply to Peter ZijlstraSaner PERF_CAPABILITIES support, which also exposes pebs_trap. Use that latter to make PEBS's use of LBR conditional since a fault-like pebs should already report the correct IP.
(As of this writing there is no known hardware that implements !pebs_trap)
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 15 ++++++++++++-- arch/x86/kernel/cpu/perf_event_intel.c | 10 +++++++++ arch/x86/kernel/cpu/perf_event_intel_ds.c | 30 ++++++++++++++--------------- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 18 +++-------------- 4 files changed, 42 insertions(+), 31 deletions(-)
arch/x86/kernel/cpu/perf_event.c +13 -2
@@ -154,6 +154,17 @@ struct cpu_hw_events {#define for_each_event_constraint (e, c) \ for ((e) = (c); (e)->cmask; (e)++) +union perf_capabilities { +struct { +u64 lbr_format :6 ; +u64 pebs_trap :1 ; +u64 pebs_arch_reg :1 ; +u64 pebs_format :4 ; +u64 smm_freeze :1 ; + }; +u64 capabilities; +}; +/* * struct x86_pmu - generic x86 pmu */
@@ -190,7 +201,8 @@ struct x86_pmu {/* * Intel Arch Perfmon v2+ */ -u64 intel_ctrl; +u64 intel_ctrl; +union perf_capabilities intel_cap;/* * Intel DebugStore bits
@@ -205,7 +217,6 @@ struct x86_pmu {*/ unsigned long lbr_tos, lbr_from, lbr_to;/* MSR base regs */ int lbr_nr;/* hardware stack size */ -int lbr_format;/* hardware format */ };static struct x86_pmu x86_pmu__read_mostly ;
arch/x86/kernel/cpu/perf_event_intel.c +10 -0
@@ -834,6 +834,16 @@ static __init int intel_pmu_init(void)if (version >1 ) x86_pmu.num_events_fixed =max ((int )edx.split.num_events_fixed,3 ); +/* +* v2 and above have a perf capabilities MSR +*/ +if (version >1 ) { +u64 capabilities; + +rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); + x86_pmu.intel_cap.capabilities = capabilities; + } +intel_ds_init ();/*
arch/x86/kernel/cpu/perf_event_intel_ds.c +15 -15
@@ -342,7 +342,8 @@ static void intel_pmu_pebs_enable(struct val |=1ULL << hwc->idx;wrmsrl (MSR_IA32_PEBS_ENABLE , val); -intel_pmu_lbr_enable (event); +if (x86_pmu.intel_cap.pebs_trap) +intel_pmu_lbr_enable (event); }static void intel_pmu_pebs_disable (struct perf_event *event)
@@ -356,7 +357,8 @@ static void intel_pmu_pebs_disable(struc hwc->config |=ARCH_PERFMON_EVENTSEL_INT ; -intel_pmu_lbr_disable (event); +if (x86_pmu.intel_cap.pebs_trap) +intel_pmu_lbr_disable (event); }static void intel_pmu_pebs_enable_all (void )
@@ -395,6 +397,12 @@ static int intel_pmu_pebs_fixup_ip(strucunsigned long old_to, to = cpuc->lbr_entries[0 ].to;unsigned long ip = regs->ip; +/* +* We don't need to fixup if the PEBS assist is fault like +*/ +if (!x86_pmu.intel_cap.pebs_trap) +return 1 ; +if (!cpuc->lbr_stack.nr || !from || !to)return 0 ;
@@ -599,34 +607,26 @@ static void intel_ds_init(void) x86_pmu.bts =boot_cpu_has (X86_FEATURE_BTS ); x86_pmu.pebs =boot_cpu_has (X86_FEATURE_PEBS );if (x86_pmu.pebs) { -int format =0 ; - -if (x86_pmu.version >1 ) { -u64 capabilities; -/* -* v2+ has a PEBS format field -*/ -rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); - format = (capabilities >>8 ) &0xf ; - } +char pebs_type = x86_pmu.intel_cap.pebs_trap ?'+' :'-' ; +int format = x86_pmu.intel_cap.pebs_format;switch (format) {case 0 : -printk (KERN_CONT "PEBS v0, " ); +printk (KERN_CONT "PEBS fmt0%c, " , pebs_type); x86_pmu.pebs_record_size =sizeof (struct pebs_record_core ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_core; x86_pmu.pebs_constraints = intel_core_pebs_events;break ;case 1 : -printk (KERN_CONT "PEBS v1, " ); +printk (KERN_CONT "PEBS fmt1%c, " , pebs_type); x86_pmu.pebs_record_size =sizeof (struct pebs_record_nhm ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm; x86_pmu.pebs_constraints = intel_nehalem_pebs_events;break ;default : -printk (KERN_CONT "PEBS unknown format: %d, " , format); +printk (KERN_CONT "no PEBS fmt%d%c, " , format, pebs_type); x86_pmu.pebs =0 ;break ; }
arch/x86/kernel/cpu/perf_event_intel_lbr.c +4 -14
@@ -53,7 +53,7 @@ static void intel_pmu_lbr_reset_64(void)static void intel_pmu_lbr_reset (void ) { -if (x86_pmu.lbr_format ==LBR_FORMAT_32 ) +if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_32 )intel_pmu_lbr_reset_32 ();else intel_pmu_lbr_reset_64 ();
@@ -155,6 +155,7 @@ static void intel_pmu_lbr_read_32(structstatic void intel_pmu_lbr_read_64 (struct cpu_hw_events *cpuc) {unsigned long mask = x86_pmu.lbr_nr -1 ; +int lbr_format = x86_pmu.intel_cap.lbr_format;u64 tos =intel_pmu_lbr_tos ();int i;
@@ -165,7 +166,7 @@ static void intel_pmu_lbr_read_64(structrdmsrl (x86_pmu.lbr_from + lbr_idx, from);rdmsrl (x86_pmu.lbr_to + lbr_idx, to); -if (x86_pmu.lbr_format ==LBR_FORMAT_EIP_FLAGS ) { +if (lbr_format ==LBR_FORMAT_EIP_FLAGS ) { flags = !!(from &LBR_FROM_FLAG_MISPRED ); from = (u64)((((s64 )from) <<1 ) >>1 ); }
@@ -184,23 +185,14 @@ static void intel_pmu_lbr_read(void)if (!cpuc->lbr_users)return ; -if (x86_pmu.lbr_format ==LBR_FORMAT_32 ) +if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_32 )intel_pmu_lbr_read_32 (cpuc);else intel_pmu_lbr_read_64 (cpuc); } -static int intel_pmu_lbr_format (void ) -{ -u64 capabilities; - -rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); -return capabilities &0x1f ; -} -static void intel_pmu_lbr_init_core (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =4 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x40 ;
@@ -209,7 +201,6 @@ static void intel_pmu_lbr_init_core(voidstatic void intel_pmu_lbr_init_nhm (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =16 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x680 ;
@@ -218,7 +209,6 @@ static void intel_pmu_lbr_init_nhm(void)static void intel_pmu_lbr_init_atom (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =8 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x40 ;
[tip:perf/pebs] perf, x86: Clean up IA32_PERF_CAPABILITIES usage
Commit-ID: 8db909a7e3c888b5d45aef7650d74ccebe3ce725 Gitweb: http://git.kernel.org/tip/8db909a7e3c888b5d45aef7650d74ccebe3ce725 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 3 Mar 2010 17:07:40 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:33 +0100
perf, x86: Clean up IA32_PERF_CAPABILITIES usage
Saner PERF_CAPABILITIES support, which also exposes pebs_trap. Use that latter to make PEBS's use of LBR conditional since a fault-like pebs should already report the correct IP.
( As of this writing there is no known hardware that implements
!pebs_trap )
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event.c | 15 ++++++++++++- arch/x86/kernel/cpu/perf_event_intel.c | 10 +++++++++ arch/x86/kernel/cpu/perf_event_intel_ds.c | 30 ++++++++++++++-------------- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 18 +++------------- 4 files changed, 42 insertions(+), 31 deletions(-)
arch/x86/kernel/cpu/perf_event.c +13 -2
@@ -154,6 +154,17 @@ struct cpu_hw_events {#define for_each_event_constraint (e, c) \ for ((e) = (c); (e)->cmask; (e)++) +union perf_capabilities { +struct { +u64 lbr_format :6 ; +u64 pebs_trap :1 ; +u64 pebs_arch_reg :1 ; +u64 pebs_format :4 ; +u64 smm_freeze :1 ; + }; +u64 capabilities; +}; +/* * struct x86_pmu - generic x86 pmu */
@@ -195,7 +206,8 @@ struct x86_pmu {/* * Intel Arch Perfmon v2+ */ -u64 intel_ctrl; +u64 intel_ctrl; +union perf_capabilities intel_cap;/* * Intel DebugStore bits
@@ -210,7 +222,6 @@ struct x86_pmu {*/ unsigned long lbr_tos, lbr_from, lbr_to;/* MSR base regs */ int lbr_nr;/* hardware stack size */ -int lbr_format;/* hardware format */ };static struct x86_pmu x86_pmu__read_mostly ;
arch/x86/kernel/cpu/perf_event_intel.c +10 -0
@@ -835,6 +835,16 @@ static __init int intel_pmu_init(void)if (version >1 ) x86_pmu.num_events_fixed =max ((int )edx.split.num_events_fixed,3 ); +/* +* v2 and above have a perf capabilities MSR +*/ +if (version >1 ) { +u64 capabilities; + +rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); + x86_pmu.intel_cap.capabilities = capabilities; + } +intel_ds_init ();/*
arch/x86/kernel/cpu/perf_event_intel_ds.c +15 -15
@@ -342,7 +342,8 @@ static void intel_pmu_pebs_enable(struct perf_event *event) val |=1ULL << hwc->idx;wrmsrl (MSR_IA32_PEBS_ENABLE , val); -intel_pmu_lbr_enable (event); +if (x86_pmu.intel_cap.pebs_trap) +intel_pmu_lbr_enable (event); }static void intel_pmu_pebs_disable (struct perf_event *event)
@@ -356,7 +357,8 @@ static void intel_pmu_pebs_disable(struct perf_event *event) hwc->config |=ARCH_PERFMON_EVENTSEL_INT ; -intel_pmu_lbr_disable (event); +if (x86_pmu.intel_cap.pebs_trap) +intel_pmu_lbr_disable (event); }static void intel_pmu_pebs_enable_all (void )
@@ -395,6 +397,12 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)unsigned long old_to, to = cpuc->lbr_entries[0 ].to;unsigned long ip = regs->ip; +/* +* We don't need to fixup if the PEBS assist is fault like +*/ +if (!x86_pmu.intel_cap.pebs_trap) +return 1 ; +if (!cpuc->lbr_stack.nr || !from || !to)return 0 ;
@@ -589,34 +597,26 @@ static void intel_ds_init(void) x86_pmu.bts =boot_cpu_has (X86_FEATURE_BTS ); x86_pmu.pebs =boot_cpu_has (X86_FEATURE_PEBS );if (x86_pmu.pebs) { -int format =0 ; - -if (x86_pmu.version >1 ) { -u64 capabilities; -/* -* v2+ has a PEBS format field -*/ -rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); - format = (capabilities >>8 ) &0xf ; - } +char pebs_type = x86_pmu.intel_cap.pebs_trap ?'+' :'-' ; +int format = x86_pmu.intel_cap.pebs_format;switch (format) {case 0 : -printk (KERN_CONT "PEBS v0, " ); +printk (KERN_CONT "PEBS fmt0%c, " , pebs_type); x86_pmu.pebs_record_size =sizeof (struct pebs_record_core ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_core; x86_pmu.pebs_constraints = intel_core_pebs_events;break ;case 1 : -printk (KERN_CONT "PEBS v1, " ); +printk (KERN_CONT "PEBS fmt1%c, " , pebs_type); x86_pmu.pebs_record_size =sizeof (struct pebs_record_nhm ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm; x86_pmu.pebs_constraints = intel_nehalem_pebs_events;break ;default : -printk (KERN_CONT "PEBS unknown format: %d, " , format); +printk (KERN_CONT "no PEBS fmt%d%c, " , format, pebs_type); x86_pmu.pebs =0 ;break ; }
arch/x86/kernel/cpu/perf_event_intel_lbr.c +4 -14
@@ -53,7 +53,7 @@ static void intel_pmu_lbr_reset_64(void)static void intel_pmu_lbr_reset (void ) { -if (x86_pmu.lbr_format ==LBR_FORMAT_32 ) +if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_32 )intel_pmu_lbr_reset_32 ();else intel_pmu_lbr_reset_64 ();
@@ -155,6 +155,7 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)static void intel_pmu_lbr_read_64 (struct cpu_hw_events *cpuc) {unsigned long mask = x86_pmu.lbr_nr -1 ; +int lbr_format = x86_pmu.intel_cap.lbr_format;u64 tos =intel_pmu_lbr_tos ();int i;
@@ -165,7 +166,7 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)rdmsrl (x86_pmu.lbr_from + lbr_idx, from);rdmsrl (x86_pmu.lbr_to + lbr_idx, to); -if (x86_pmu.lbr_format ==LBR_FORMAT_EIP_FLAGS ) { +if (lbr_format ==LBR_FORMAT_EIP_FLAGS ) { flags = !!(from &LBR_FROM_FLAG_MISPRED ); from = (u64)((((s64 )from) <<1 ) >>1 ); }
@@ -184,23 +185,14 @@ static void intel_pmu_lbr_read(void)if (!cpuc->lbr_users)return ; -if (x86_pmu.lbr_format ==LBR_FORMAT_32 ) +if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_32 )intel_pmu_lbr_read_32 (cpuc);else intel_pmu_lbr_read_64 (cpuc); } -static int intel_pmu_lbr_format (void ) -{ -u64 capabilities; - -rdmsrl (MSR_IA32_PERF_CAPABILITIES , capabilities); -return capabilities &0x1f ; -} -static void intel_pmu_lbr_init_core (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =4 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x40 ;
@@ -209,7 +201,6 @@ static void intel_pmu_lbr_init_core(void)static void intel_pmu_lbr_init_nhm (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =16 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x680 ;
@@ -218,7 +209,6 @@ static void intel_pmu_lbr_init_nhm(void)static void intel_pmu_lbr_init_atom (void ) { - x86_pmu.lbr_format =intel_pmu_lbr_format (); x86_pmu.lbr_nr =8 ; x86_pmu.lbr_tos =0x01c9 ; x86_pmu.lbr_from =0x40 ;
PATCH 12/14 perf, x86: Expose the full PEBS record using PERF_SAMPLE_RAW
In reply to Peter ZijlstraExpose the full PEBS record using PERF_SAMPLE_RAW
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event_intel_ds.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
arch/x86/kernel/cpu/perf_event_intel_ds.c +14 -0
@@ -457,6 +457,7 @@ static void intel_pmu_drain_pebs_core(ststruct perf_event *event = cpuc->events[0 ];/* PMC0 only */ struct pebs_record_core *at, *top;struct perf_sample_data data; +struct perf_raw_record raw;struct pt_regs regs;int n;
@@ -490,6 +491,12 @@ static void intel_pmu_drain_pebs_core(st at += n-1 ; } +if (event->attr.sample_type &PERF_SAMPLE_RAW ) { + raw.size = x86_pmu.pebs_record_size; + raw.data = at; + data.raw = &raw; + } +/* * We use the interrupt regs as a base because the PEBS record * does not contain a full regs set, specifically it seems to
@@ -524,6 +531,7 @@ static void intel_pmu_drain_pebs_nhm(strstruct pebs_record_nhm *at, *top;struct perf_sample_data data;struct perf_event *event =NULL ; +struct perf_raw_record raw;struct pt_regs regs;int bit, n;
@@ -572,6 +580,12 @@ static void intel_pmu_drain_pebs_nhm(strperf_sample_data_init (&data,0 ); data.period = event->hw.last_period; +if (event->attr.sample_type &PERF_SAMPLE_RAW ) { + raw.size = x86_pmu.pebs_record_size; + raw.data = at; + data.raw = &raw; + } +/* * See the comment in intel_pmu_drain_pebs_core() */
[tip:perf/pebs] perf, x86: Expose the full PEBS record using PERF_SAMPLE_RAW
Commit-ID: 7e1a40dda619b0483fbe0740494ed2c2a1f05289 Gitweb: http://git.kernel.org/tip/7e1a40dda619b0483fbe0740494ed2c2a1f05289 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 4 Mar 2010 12:38:03 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:33 +0100
perf, x86: Expose the full PEBS record using PERF_SAMPLE_RAW
Expose the full PEBS record using PERF_SAMPLE_RAW
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_event_intel_ds.c | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_ds.c +14 -0
@@ -457,6 +457,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)struct perf_event *event = cpuc->events[0 ];/* PMC0 only */ struct pebs_record_core *at, *top;struct perf_sample_data data; +struct perf_raw_record raw;struct pt_regs regs;int n;
@@ -479,6 +480,12 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; +if (event->attr.sample_type &PERF_SAMPLE_RAW ) { + raw.size = x86_pmu.pebs_record_size; + raw.data = at; + data.raw = &raw; + } + n = top - at;/*
@@ -521,6 +528,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)struct pebs_record_nhm *at, *top;struct perf_sample_data data;struct perf_event *event =NULL ; +struct perf_raw_record raw;struct pt_regs regs;int bit, n;
@@ -562,6 +570,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)perf_sample_data_init (&data,0 ); data.period = event->hw.last_period; +if (event->attr.sample_type &PERF_SAMPLE_RAW ) { + raw.size = x86_pmu.pebs_record_size; + raw.data = at; + data.raw = &raw; + } +/* * See the comment in intel_pmu_drain_pebs_core() */
PATCH 13/14 x86: Move MAX_INSN_SIZE into asm/insn.h
In reply to Peter ZijlstraSince there's now two users for this, place it in a common header.
LKML-Reference: <new-submission>
--- arch/x86/include/asm/insn.h | 2 ++ arch/x86/include/asm/kprobes.h | 2 +- arch/x86/kernel/cpu/perf_event_intel_ds.c | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-)
arch/x86/include/asm/insn.h +2 -0
@@ -68,6 +68,8 @@ struct insn {const insn_byte_t *next_byte; }; +#define MAX_INSN_SIZE 16 +#define X86_MODRM_MOD (modrm) (((modrm) & 0xc0) >> 6)#define X86_MODRM_REG (modrm) (((modrm) & 0x38) >> 3)#define X86_MODRM_RM (modrm) ((modrm) & 0x07)
arch/x86/include/asm/kprobes.h +1 -1
@@ -24,6 +24,7 @@#include <linux/types.h> #include <linux/ptrace.h> #include <linux/percpu.h> +#include <asm/insn.h> #define __ARCH_WANT_KPROBES_INSN_SLOT
@@ -36,7 +37,6 @@ typedef u8 kprobe_opcode_t;#define RELATIVEJUMP_SIZE 5#define RELATIVECALL_OPCODE 0xe8#define RELATIVE_ADDR_SIZE 4 -#define MAX_INSN_SIZE 16#define MAX_STACK_SIZE 64#define MIN_STACK_SIZE (ADDR ) \ (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \
arch/x86/kernel/cpu/perf_event_intel_ds.c +0 -2
@@ -379,8 +379,6 @@ static void intel_pmu_pebs_disable_all(v#include <asm/insn.h> -#define MAX_INSN_SIZE 16 -static inline bool kernel_ip (unsigned long ip) {#ifdef CONFIG_X86_32
Re: [PATCH 13/14] x86: Move MAX_INSN_SIZE into asm/insn.h
Quoted 5 lines · Peter Zijlstra
Since there's now two users for this, place it in a common header.
CC Masami Hiramatsu <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>LKML-Reference: <new-submission>
Thank you,
Signature · 2 lines
Masami Hiramatsu e-mail: [email protected]
[tip:perf/pebs] x86: Move MAX_INSN_SIZE into asm/insn.h
In reply to Peter ZijlstraCommit-ID: 30a813ae035d3e220a89609adce878e045c49547 Gitweb: http://git.kernel.org/tip/30a813ae035d3e220a89609adce878e045c49547 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 4 Mar 2010 13:49:21 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Wed, 10 Mar 2010 13:23:34 +0100
x86: Move MAX_INSN_SIZE into asm/insn.h
Since there's now two users for this, place it in a common header.
LKML-Reference: <[email protected]>
--- arch/x86/include/asm/insn.h | 2 ++ arch/x86/include/asm/kprobes.h | 2 +- arch/x86/kernel/cpu/perf_event_intel_ds.c | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-)
arch/x86/include/asm/insn.h +2 -0
@@ -68,6 +68,8 @@ struct insn {const insn_byte_t *next_byte; }; +#define MAX_INSN_SIZE 16 +#define X86_MODRM_MOD (modrm) (((modrm) & 0xc0) >> 6)#define X86_MODRM_REG (modrm) (((modrm) & 0x38) >> 3)#define X86_MODRM_RM (modrm) ((modrm) & 0x07)
arch/x86/include/asm/kprobes.h +1 -1
@@ -24,6 +24,7 @@#include <linux/types.h> #include <linux/ptrace.h> #include <linux/percpu.h> +#include <asm/insn.h> #define __ARCH_WANT_KPROBES_INSN_SLOT
@@ -36,7 +37,6 @@ typedef u8 kprobe_opcode_t;#define RELATIVEJUMP_SIZE 5#define RELATIVECALL_OPCODE 0xe8#define RELATIVE_ADDR_SIZE 4 -#define MAX_INSN_SIZE 16#define MAX_STACK_SIZE 64#define MIN_STACK_SIZE (ADDR ) \ (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \
arch/x86/kernel/cpu/perf_event_intel_ds.c +0 -2
@@ -379,8 +379,6 @@ static void intel_pmu_pebs_disable_all(void)#include <asm/insn.h> -#define MAX_INSN_SIZE 16 -static inline bool kernel_ip (unsigned long ip) {#ifdef CONFIG_X86_32
PATCH 14/14 perf, x86: Implement PERF_SAMPLE_BRANCH_STACK
In reply to Peter ZijlstraNot for merging until there's a sensible use case implemented in tools/perf as well.
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 3 +- arch/x86/kernel/cpu/perf_event_intel.c | 10 ++++++- arch/x86/kernel/cpu/perf_event_intel_ds.c | 20 ++++++--------- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 4 ++- include/linux/perf_event.h | 15 ++++++++--- kernel/perf_event.c | 38 ++++++++++++++++++++++------- 6 files changed, 62 insertions(+), 28 deletions(-)
include/linux/perf_event.h +11 -4
@@ -125,8 +125,9 @@ enum perf_event_sample_format {PERF_SAMPLE_PERIOD =1U <<8 ,PERF_SAMPLE_STREAM_ID =1U <<9 ,PERF_SAMPLE_RAW =1U <<10 , +PERF_SAMPLE_BRANCH_STACK =1U <<11 , - PERF_SAMPLE_MAX = 1U << 11, /* non-ABI */ + PERF_SAMPLE_MAX = 1U << 12, /* non-ABI */ };/*
@@ -399,9 +400,13 @@ enum perf_event_type {* * { struct read_format values; } && PERF_SAMPLE_READ * -* { u64 nr, +* { u64 nr; * u64 ips[nr]; } && PERF_SAMPLE_CALLCHAIN * +* { u64 nr; +* { u64 from, to, flags; +* } lbr[nr]; } && PERF_SAMPLE_BRANCH_STACK +* * # * # The RAW record below is opaque data wrt the ABI * #
@@ -817,13 +822,15 @@ struct perf_sample_data {u64 period;struct perf_callchain_entry *callchain;struct perf_raw_record *raw; +struct perf_branch_stack *branches; };static inline void perf_sample_data_init (struct perf_sample_data *data,u64 addr) { - data->addr = addr; - data->raw =NULL ; + data->addr = addr; + data->raw =NULL ; + data->branches =NULL ; } extern void perf_output_sample(struct perf_output_handle *handle,
kernel/perf_event.c +30 -8
@@ -3178,12 +3178,9 @@ void perf_output_sample(struct perf_outpif (sample_type &PERF_SAMPLE_CALLCHAIN ) {if (data->callchain) { -int size =1 ; +int size =sizeof (u64); -if (data->callchain) - size += data->callchain->nr; - - size *=sizeof (u64); + size += data->callchain->nr *sizeof (u64);perf_output_copy (handle, data->callchain, size); }else {
@@ -3192,6 +3189,20 @@ void perf_output_sample(struct perf_outp } } +if (sample_type &PERF_SAMPLE_BRANCH_STACK ) { +if (data->branches) { +int size =sizeof (u64); + + size += data->branches->nr * +sizeof (struct perf_branch_entry ); + +perf_output_copy (handle, data->branches, size); + }else { +u64 nr =0 ; +perf_output_put (handle, nr); + } + } +if (sample_type &PERF_SAMPLE_RAW ) {if (data->raw) {perf_output_put (handle, data->raw->size);
@@ -3274,14 +3285,25 @@ void perf_prepare_sample(struct perf_eve header->size +=perf_event_read_size (event);if (sample_type &PERF_SAMPLE_CALLCHAIN ) { -int size =1 ; +int size =sizeof (u64); data->callchain =perf_callchain (regs);if (data->callchain) - size += data->callchain->nr; + size += data->callchain->nr *sizeof (u64); + + header->size += size; + } - header->size += size *sizeof (u64); +if (sample_type &PERF_SAMPLE_BRANCH_STACK ) { +int size =sizeof (u64); + +if (data->branches) { + size += data->branches->nr * +sizeof (struct perf_branch_entry ); + } + + header->size += size; }if (sample_type &PERF_SAMPLE_RAW ) {
arch/x86/kernel/cpu/perf_event_intel.c +8 -2
@@ -548,6 +548,9 @@ static void intel_pmu_disable_event(struif (unlikely (event->attr.precise))intel_pmu_pebs_disable (event); + +if (event->attr.sample_type &PERF_SAMPLE_BRANCH_STACK ) +intel_pmu_lbr_disable (event); }static void intel_pmu_enable_fixed (struct hw_perf_event *hwc)
@@ -602,6 +605,9 @@ static void intel_pmu_enable_event(strucif (unlikely (event->attr.precise))intel_pmu_pebs_enable (event); +if (event->attr.sample_type &PERF_SAMPLE_BRANCH_STACK ) +intel_pmu_lbr_enable (event); +__x86_pmu_enable_event (hwc); }
@@ -677,13 +683,13 @@ again:inc_irq_stat (apic_perf_irqs); ack = status; -intel_pmu_lbr_read (); +intel_pmu_lbr_read (&data);/* * PEBS overflow sets bit 62 in the global status register */ if (__test_and_clear_bit (62 , (unsigned long *)&status)) - x86_pmu.drain_pebs (regs); + x86_pmu.drain_pebs (&data, regs);for_each_bit (bit, (unsigned long *)&status,X86_PMC_IDX_MAX ) {struct perf_event *event = cpuc->events[bit];
arch/x86/kernel/cpu/perf_event_intel_ds.c +8 -12
@@ -448,13 +448,12 @@ static int intel_pmu_pebs_fixup_ip(strucstatic int intel_pmu_save_and_restart (struct perf_event *event);static void intel_pmu_disable_event (struct perf_event *event); -static void intel_pmu_drain_pebs_core (struct pt_regs *iregs) +static void intel_pmu_drain_pebs_core (struct perf_sample_data *data,struct pt_regs *iregs) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct debug_store *ds = cpuc->ds;struct perf_event *event = cpuc->events[0 ];/* PMC0 only */ struct pebs_record_core *at, *top; -struct perf_sample_data data;struct perf_raw_record raw;struct pt_regs regs;int n;
@@ -475,8 +474,7 @@ static void intel_pmu_drain_pebs_core(stif (!intel_pmu_save_and_restart (event)) goto out; -perf_sample_data_init (&data,0 ); - data.period = event->hw.last_period; + data->period = event->hw.last_period; n = top - at;
@@ -492,7 +490,7 @@ static void intel_pmu_drain_pebs_core(stif (event->attr.sample_type &PERF_SAMPLE_RAW ) { raw.size = x86_pmu.pebs_record_size; raw.data = at; - data.raw = &raw; + data->raw = &raw; }/*
@@ -515,19 +513,18 @@ static void intel_pmu_drain_pebs_core(stelse regs.flags &= ~PERF_EFLAGS_EXACT ; -if (perf_event_overflow (event,1 , &data, ®s)) +if (perf_event_overflow (event,1 , data, ®s))intel_pmu_disable_event (event); out:intel_pmu_pebs_enable_all (); } -static void intel_pmu_drain_pebs_nhm (struct pt_regs *iregs) +static void intel_pmu_drain_pebs_nhm (struct perf_sample_data *data,struct pt_regs *iregs) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);struct debug_store *ds = cpuc->ds;struct pebs_record_nhm *at, *top; -struct perf_sample_data data;struct perf_event *event =NULL ;struct perf_raw_record raw;struct pt_regs regs;
@@ -575,13 +572,12 @@ static void intel_pmu_drain_pebs_nhm(strif (!intel_pmu_save_and_restart (event))continue ; -perf_sample_data_init (&data,0 ); - data.period = event->hw.last_period; + data->period = event->hw.last_period;if (event->attr.sample_type &PERF_SAMPLE_RAW ) { raw.size = x86_pmu.pebs_record_size; raw.data = at; - data.raw = &raw; + data->raw = &raw; }/*
@@ -597,7 +593,7 @@ static void intel_pmu_drain_pebs_nhm(strelse regs.flags &= ~PERF_EFLAGS_EXACT ; -if (perf_event_overflow (event,1 , &data, ®s)) +if (perf_event_overflow (event,1 , data, ®s))intel_pmu_disable_event (event); } out:
arch/x86/kernel/cpu/perf_event.c +2 -1
@@ -209,7 +209,8 @@ struct x86_pmu {*/ int bts, pebs;int pebs_record_size; -void (*drain_pebs)(struct pt_regs *regs); +void (*drain_pebs)(struct perf_sample_data *data, +struct pt_regs *regs);struct event_constraint *pebs_constraints;/*
arch/x86/kernel/cpu/perf_event_intel_lbr.c +3 -1
@@ -178,7 +178,7 @@ static void intel_pmu_lbr_read_64(struct cpuc->lbr_stack.nr = i; } -static void intel_pmu_lbr_read (void ) +static void intel_pmu_lbr_read (struct perf_sample_data *data) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -189,6 +189,8 @@ static void intel_pmu_lbr_read(void)intel_pmu_lbr_read_32 (cpuc); elseintel_pmu_lbr_read_64 (cpuc ); + + data->branches = &cpuc->lbr_stack; }static void intel_pmu_lbr_init_core (void )