perf, core: simplify need branch stack check
Subsystems (6)
PATCH v5 10/16 perf, core: simplify need branch stack check
From: Kan Liang <[email protected]>
event->attr.branch_sample_type is non-zero no matter branch stack is enabled explicitly or is enabled implicitly. we can use it to replace intel_pmu_needs_lbr_smpl(). This avoids duplicating code that implicitly enables the LBR.
--- arch/x86/kernel/cpu/perf_event_intel.c | 20 +++----------------- include/linux/perf_event.h | 5 +++++ kernel/events/core.c | 3 +++ 3 files changed, 11 insertions(+), 17 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +3 -17
@@ -1029,20 +1029,6 @@ static __initconst const u64 slm_hw_cache_event_ids }, }; -static inline bool intel_pmu_needs_lbr_smpl (struct perf_event *event) -{ -/* user explicitly requested branch sampling */ -if (has_branch_stack (event)) -return true; - -/* implicit branch sampling to correct PEBS skid */ -if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip >1 && - x86_pmu.intel_cap.pebs_format <2 ) -return true; - -return false; -} -static void intel_pmu_disable_all (void ) {struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events);
@@ -1207,7 +1193,7 @@ static void intel_pmu_disable_event(struct perf_event *event)* must disable before any actual event * because any event may be combined with LBR */ -if (intel_pmu_needs_lbr_smpl (event)) +if (needs_branch_stack (event))intel_pmu_lbr_disable (event);if (unlikely (hwc->config_base ==MSR_ARCH_PERFMON_FIXED_CTR_CTRL )) {
@@ -1268,7 +1254,7 @@ static void intel_pmu_enable_event(struct perf_event *event)* must enabled before any actual event * because any event may be combined with LBR */ -if (intel_pmu_needs_lbr_smpl (event)) +if (needs_branch_stack (event))intel_pmu_lbr_enable (event);if (event->attr.exclude_host)
@@ -1747,7 +1733,7 @@ static int intel_pmu_hw_config(struct perf_event *event)if (event->attr.precise_ip && x86_pmu.pebs_aliases) x86_pmu.pebs_aliases (event); -if (intel_pmu_needs_lbr_smpl (event)) { +if (needs_branch_stack (event)) { ret =intel_pmu_setup_lbr_filter (event);if (ret)return ret;
include/linux/perf_event.h +5 -0
@@ -792,6 +792,11 @@ static inline bool has_branch_stack(struct perf_event *event)return event->attr.sample_type &PERF_SAMPLE_BRANCH_STACK ; } +static inline bool needs_branch_stack (struct perf_event *event) +{ +return event->attr.branch_sample_type !=0 ; +} +extern int perf_output_begin (struct perf_output_handle *handle,struct perf_event *event,unsigned int size);extern void perf_output_end (struct perf_output_handle *handle);
kernel/events/core.c +3 -0
@@ -7003,6 +7003,9 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,if (attr->inherit && (attr->read_format &PERF_FORMAT_GROUP )) goto err_ns; +if (!has_branch_stack (event)) + event->attr.branch_sample_type =0 ; + pmu =perf_init_event (event);if (!pmu) goto err_ns;
Signature · 1 line
1.8.3.2
PATCH v5 11/16 perf, core: Pass perf_sample_data to perf_callchain()
From: Kan Liang <[email protected]>
Haswell has a new feature that utilizes the existing Last Branch Record facility to record call chains. When the feature is enabled, function call will be collected as normal, but as return instructions are executed the last captured branch record is popped from the on-chip LBR registers.
The LBR call stack facility can help perf to get call chains of progam without frame pointer.
This patch modifies various architectures' perf_callchain() to accept perf sample data. Later patch will add code that use the sample data to get call chains.
--- arch/arm/kernel/perf_event.c | 4 ++-- arch/powerpc/perf/callchain.c | 4 ++-- arch/sparc/kernel/perf_event.c | 4 ++-- arch/x86/kernel/cpu/perf_event.c | 4 ++-- include/linux/perf_event.h | 4 +++- kernel/events/callchain.c | 8 +++++--- kernel/events/core.c | 2 +- kernel/events/internal.h | 3 ++- 8 files changed, 19 insertions(+), 14 deletions(-)
arch/arm/kernel/perf_event.c +2 -2
@@ -584,8 +584,8 @@ user_backtrace(struct frame_tail __user *tail,return buftail.fp -1 ; } -void -perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs) +void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {struct frame_tail __user *tail;
arch/powerpc/perf/callchain.c +2 -2
@@ -482,8 +482,8 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, } } -void -perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs) +void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {if (current_is_64bit ())perf_callchain_user_64 (entry, regs);
arch/sparc/kernel/perf_event.c +2 -2
@@ -1791,8 +1791,8 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, }while (entry->nr <PERF_MAX_STACK_DEPTH ); } -void -perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs) +void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {perf_callchain_store (entry, regs->tpc);
arch/x86/kernel/cpu/perf_event.c +2 -2
@@ -2049,8 +2049,8 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry) }#endif -void -perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs) +void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {struct stack_frame frame;const void __user *fp;
include/linux/perf_event.h +3 -1
@@ -735,7 +735,9 @@ extern void perf_event_fork(struct task_struct *tsk);/* Callchains */ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); -extern void perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs); +extern void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs, +struct perf_sample_data *data);extern void perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs);static inline void perf_callchain_store (struct perf_callchain_entry *entry,u64 ip)
kernel/events/callchain.c +5 -3
@@ -30,7 +30,8 @@ __weak void perf_callchain_kernel(struct perf_callchain_entry *entry, }__weak void perf_callchain_user (struct perf_callchain_entry *entry, -struct pt_regs *regs) +struct pt_regs *regs, +struct perf_sample_data *data) { }
@@ -157,7 +158,8 @@ put_callchain_entry(int rctx) }struct perf_callchain_entry * -perf_callchain (struct perf_event *event,struct pt_regs *regs) +perf_callchain (struct perf_event *event,struct pt_regs *regs, +struct perf_sample_data *data) {int rctx;struct perf_callchain_entry *entry;
@@ -198,7 +200,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) goto exit_put;perf_callchain_store (entry,PERF_CONTEXT_USER ); -perf_callchain_user (entry, regs); +perf_callchain_user (entry, regs, data); } }
kernel/events/core.c +1 -1
@@ -4885,7 +4885,7 @@ void perf_prepare_sample(struct perf_event_header *header,if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; - data->callchain =perf_callchain (event, regs); + data->callchain =perf_callchain (event, regs, data);if (data->callchain) size += data->callchain->nr;
kernel/events/internal.h +2 -1
@@ -147,7 +147,8 @@ DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)/* Callchain handling */ extern struct perf_callchain_entry * -perf_callchain (struct perf_event *event,struct pt_regs *regs); +perf_callchain (struct perf_event *event,struct pt_regs *regs, +struct perf_sample_data *data);extern int get_callchain_buffers (void );extern void put_callchain_buffers (void );
Signature · 1 line
1.8.3.2
PATCH v5 12/16 perf, x86: use LBR call stack to get user callchain
In reply to [email protected]From: Kan Liang <[email protected]>
Haswell has a new feature that utilizes the existing Last Branch Record facility to record call chains. When the feature is enabled, function call will be collected as normal, but as return instructions are executed
the last captured branch record is popped from the on-chip LBR registers.
The LBR call stack facility can help perf to get call chains of progam without frame pointer.
This patch makes x86's perf_callchain_user() failback to use LBR call stack data when there is no frame pointer in the user program. The 'from'
address of branch entry is used as 'return' address of function call.
--- arch/x86/kernel/cpu/perf_event.c | 34 ++++++++++++++++++++++++++---- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 2 ++ include/linux/perf_event.h | 1 + 4 files changed, 34 insertions(+), 5 deletions(-)
arch/x86/kernel/cpu/perf_event.c +30 -4
@@ -2005,12 +2005,29 @@ static unsigned long get_segment_base(unsigned int segment)return get_desc_base (desc + idx); } +static inline void +perf_callchain_lbr_callstack (struct perf_callchain_entry *entry, +struct perf_sample_data *data) +{ +struct perf_branch_stack *br_stack = data->br_stack; + +if (br_stack && br_stack->user_callstack) { +int i =0 ; + +while (i < br_stack->nr && entry->nr <PERF_MAX_STACK_DEPTH ) { +perf_callchain_store (entry, br_stack->entries[i].from); + i++; + } + } +} +#ifdef CONFIG_COMPAT #include <asm/compat.h> static inline int -perf_callchain_user32 (struct pt_regs *regs,struct perf_callchain_entry *entry) +perf_callchain_user32 (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {/* 32-bit process in 64-bit kernel. */ unsigned long ss_base, cs_base;
@@ -2039,11 +2056,16 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)perf_callchain_store (entry, cs_base + frame.return_address); fp = compat_ptr(ss_base + frame.next_frame); } + +if (fp ==compat_ptr (regs->bp)) +perf_callchain_lbr_callstack (entry, data); +return 1 ; }#else static inline int -perf_callchain_user32 (struct pt_regs *regs,struct perf_callchain_entry *entry) +perf_callchain_user32 (struct perf_callchain_entry *entry, +struct pt_regs *regs,struct perf_sample_data *data) {return 0 ; }
@@ -2073,12 +2095,12 @@ void perf_callchain_user(struct perf_callchain_entry *entry,if (!current->mm)return ; -if (perf_callchain_user32 (regs, entry)) +if (perf_callchain_user32 (entry, regs, data))return ;while (entry->nr <PERF_MAX_STACK_DEPTH ) {unsigned long bytes; - frame.next_frame =NULL ; + frame.next_frame =NULL ; frame.return_address =0 ; bytes =copy_from_user_nmi (&frame, fp,sizeof (frame));
@@ -2091,6 +2113,10 @@ void perf_callchain_user(struct perf_callchain_entry *entry,perf_callchain_store (entry, frame.return_address); fp = frame.next_frame; } + +/* try LBR callstack if there is no frame pointer */ +if (fp == (void __user *)regs->bp) +perf_callchain_lbr_callstack (entry, data); }/*
arch/x86/kernel/cpu/perf_event_intel.c +1 -1
@@ -1404,7 +1404,7 @@ again:perf_sample_data_init (&data,0 , event->hw.last_period); -if (has_branch_stack (event)) +if (needs_branch_stack (event)) data.br_stack = &cpuc->lbr_stack;if (perf_event_overflow (event, &data, regs))
arch/x86/kernel/cpu/perf_event_intel_lbr.c +2 -0
@@ -743,6 +743,8 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)int i, j, type;bool compress = false; + cpuc->lbr_stack.user_callstack =branch_user_callstack (br_sel); +/* if sampling all branches, then nothing to filter */ if ((br_sel &X86_BR_ALL ) ==X86_BR_ALL )return ;
include/linux/perf_event.h +1 -0
@@ -75,6 +75,7 @@ struct perf_raw_record {* recent branch. */ struct perf_branch_stack { +bool user_callstack;__u64 nr;struct perf_branch_entry entries[0 ]; };
Signature · 1 line
1.8.3.2
PATCH v5 13/16 perf, x86: re-organize code that implicitly enables LBR/PEBS
In reply to [email protected]From: Kan Liang <[email protected]>
make later patch more readable, no logic change.
--- arch/x86/kernel/cpu/perf_event.c | 59 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 30 deletions(-)
arch/x86/kernel/cpu/perf_event.c +29 -30
@@ -393,36 +393,35 @@ int x86_pmu_hw_config(struct perf_event *event)if (event->attr.precise_ip > precise)return -EOPNOTSUPP ; -/* -* check that PEBS LBR correction does not conflict with -* whatever the user is asking with attr->branch_sample_type -*/ -if (event->attr.precise_ip >1 && - x86_pmu.intel_cap.pebs_format <2 ) { -u64 *br_type = &event->attr.branch_sample_type; - -if (has_branch_stack (event)) { -if (!precise_br_compat (event)) -return -EOPNOTSUPP ; - -/* branch_sample_type is compatible */ - - }else { -/* -* user did not specify branch_sample_type -* -* For PEBS fixups, we capture all -* the branches at the priv level of the -* event. -*/ - *br_type =PERF_SAMPLE_BRANCH_ANY ; - -if (!event->attr.exclude_user) - *br_type |=PERF_SAMPLE_BRANCH_USER ; - -if (!event->attr.exclude_kernel) - *br_type |=PERF_SAMPLE_BRANCH_KERNEL ; - } + } +/* +* check that PEBS LBR correction does not conflict with +* whatever the user is asking with attr->branch_sample_type +*/ +if (event->attr.precise_ip >1 && x86_pmu.intel_cap.pebs_format <2 ) { +u64 *br_type = &event->attr.branch_sample_type; + +if (has_branch_stack (event)) { +if (!precise_br_compat (event)) +return -EOPNOTSUPP ; + +/* branch_sample_type is compatible */ + + }else { +/* +* user did not specify branch_sample_type +* +* For PEBS fixups, we capture all +* the branches at the priv level of the +* event. +*/ + *br_type =PERF_SAMPLE_BRANCH_ANY ; + +if (!event->attr.exclude_user) + *br_type |=PERF_SAMPLE_BRANCH_USER ; + +if (!event->attr.exclude_kernel) + *br_type |=PERF_SAMPLE_BRANCH_KERNEL ; } }
Signature · 1 line
1.8.3.2
PATCH v5 14/16 perf, x86: enable LBR callstack when recording callchain
In reply to [email protected]From: Kan Liang <[email protected]>
If a task specific event wants user space callchain but does not want branch stack sampling, enable the LBR call stack facility implicitly. The LBR call stack facility can help perf to get user space callchain in case of there is no frame pointer.
Note: this feature only affects how to get user callchain. The kernel callchain is always got by frame pointers.
--- arch/x86/kernel/cpu/perf_event.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +15 -2
@@ -423,10 +423,23 @@ int x86_pmu_hw_config(struct perf_event *event)if (!event->attr.exclude_kernel) *br_type |=PERF_SAMPLE_BRANCH_KERNEL ; } - } + }else if (x86_pmu_has_lbr_callstack () && + (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) && + !has_branch_stack (event) && + !event->attr.exclude_user && + (event->attach_state &PERF_ATTACH_TASK )) { +/* +* user did not specify branch_sample_type, +* try using the LBR call stack facility to +* record call chains of user program. +*/ + event->attr.branch_sample_type = +PERF_SAMPLE_BRANCH_USER | +PERF_SAMPLE_BRANCH_CALL_STACK ; -if (event->attr.branch_sample_type &PERF_SAMPLE_BRANCH_CALL_STACK ) +/* needs PMU specific data to save LBR stack */ event->attach_state |=PERF_ATTACH_TASK_DATA ; + }/* * Generate PMC IRQs:
Signature · 1 line
1.8.3.2
PATCH v5 15/16 perf, x86: disable FREEZE_LBRS_ON_PMI when LBR operates in callstack mode
In reply to [email protected]From: Kan Liang <[email protected]>
LBR callstack is designed for PEBS, It does not work well with FREEZE_LBRS_ON_PMI for non PEBS event. If FREEZE_LBRS_ON_PMI is set for non PEBS event, PMIs near call/return instructions may cause superfluous increase/decrease of LBR_TOS.
This patch modifies __intel_pmu_lbr_enable() to not enable FREEZE_LBRS_ON_PMI when LBR operates in callstack mode. We currently don't use LBR callstack to capture kernel space callchain, so disabling FREEZE_LBRS_ON_PMI should not be a problem.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +13 -4
@@ -131,14 +131,23 @@ static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);static void __intel_pmu_lbr_enable (void ) { -u64 debugctl;struct cpu_hw_events *cpuc = &__get_cpu_var (cpu_hw_events); +u64 debugctl, lbr_select =0 ; -if (cpuc->lbr_sel) -wrmsrl (MSR_LBR_SELECT , cpuc->lbr_sel->config); +if (cpuc->lbr_sel) { + lbr_select = cpuc->lbr_sel->config; +wrmsrl (MSR_LBR_SELECT , lbr_select); + }rdmsrl (MSR_IA32_DEBUGCTLMSR , debugctl); - debugctl |= (DEBUGCTLMSR_LBR |DEBUGCTLMSR_FREEZE_LBRS_ON_PMI ); + debugctl |=DEBUGCTLMSR_LBR ; +/* +* LBR callstack does not work well with FREEZE_LBRS_ON_PMI. +* If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions +* may cause superfluous increase/decrease of LBR_TOS. +*/ +if (!(lbr_select &LBR_CALL_STACK )) + debugctl |=DEBUGCTLMSR_FREEZE_LBRS_ON_PMI ;wrmsrl (MSR_IA32_DEBUGCTLMSR , debugctl); }
Signature · 1 line
1.8.3.2
PATCH v5 16/16 perf, x86: Discard zero length call entries in LBR call stack
In reply to [email protected]From: Kan Liang <[email protected]>
"Zero length call" uses the attribute of the call instruction to push the immediate instruction pointer on to the stack and then pops off that address into a register. This is accomplished without any matching return instruction. It confuses the hardware and make the recorded call stack incorrect.
We can partially resolve this issue by: decode call instructions and discard any zero length call entry in the LBR stack.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +11 -2
@@ -94,7 +94,8 @@ enum {X86_BR_ABORT =1 <<12 ,/* transaction abort */ X86_BR_IN_TX =1 <<13 ,/* in transaction */ X86_BR_NO_TX =1 <<14 ,/* not in transaction */ - X86_BR_CALL_STACK = 1 << 15,/* call stack */ +X86_BR_ZERO_CALL =1 <<15 ,/* zero length call */ + X86_BR_CALL_STACK = 1 << 16,/* call stack */ };#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
@@ -111,13 +112,15 @@ enum { X86_BR_JMP |\ X86_BR_IRQ |\ X86_BR_ABORT |\ - X86_BR_IND_CALL) + X86_BR_IND_CALL |\ + X86_BR_ZERO_CALL)#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)#define X86_BR_ANY_CALL \ (X86_BR_CALL |\ X86_BR_IND_CALL |\ + X86_BR_ZERO_CALL |\ X86_BR_SYSCALL |\ X86_BR_IRQ |\ X86_BR_INT)
@@ -686,6 +689,12 @@ static int branch_type(unsigned long from, unsigned long to, int abort) ret =X86_BR_INT ;break ;case 0xe8 :/* call near rel */ +insn_get_immediate (&insn); +if (insn.immediate1.value ==0 ) { +/* zero length call */ + ret =X86_BR_ZERO_CALL ; +break ; + }case 0x9a :/* call far absolute */ ret =X86_BR_CALL ;break ;
Signature · 1 line
1.8.3.2