Subsystems (2)
[Repost; I forgot to copy linux-kernel earlier. Apologies if you see it twice.]
Skylake moved to 32 Last Branch Records, from previously 16. The current call stack LBR implementation reads all LBRs and also saves/restores them on context switch. This patchkit adds some optimizations to avoid extra costs in most cases from the larger number of LBRs for call-stack, unless a nesting larger than 16 is actually needed. It applies on top of the earlier Skylake code. Some of the optimization will also benefit earlier CPUs, such as Haswell.
note: one patch is for perf user space, the rest is kernel.
-Andi
PATCH 1/5 x86, perf: Allow time stamp for free running PEBSv3
From: Andi Kleen <[email protected]>
With PEBSv3 the PEBS record contains a time stamp. That means we can allow free-running PEBS without a PMI even if the user program requested a time stamp. This avoids the need to use -T to get free running PEBS, and also avoids any problems with mis-identifying MMAPs later.
Move the free_running_flags state into a variable in x86_pmu and use it. This only works when no explicit clock_id is set.
--- arch/x86/kernel/cpu/perf_event.h | 1 + arch/x86/kernel/cpu/perf_event_intel.c | 15 ++++++++++++++- arch/x86/kernel/cpu/perf_event_intel_ds.c | 1 + 3 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/kernel/cpu/perf_event.h +1 -0
@@ -589,6 +589,7 @@ struct x86_pmu {struct event_constraint *pebs_constraints;void (*pebs_aliases)(struct perf_event *event);int max_pebs_events; +unsigned long free_running_flags;/* * Intel LBR
arch/x86/kernel/cpu/perf_event_intel.c +14 -1
@@ -2507,6 +2507,15 @@ static void intel_pebs_aliases_snb(struct perf_event *event) } } +static unsigned long intel_pmu_free_running_flags (struct perf_event *event) +{ +unsigned long flags = x86_pmu.free_running_flags; + +if (event->attr.clockid) + flags &= ~PERF_SAMPLE_TIME ; +return flags; +} +static int intel_pmu_hw_config (struct perf_event *event) {int ret =x86_pmu_hw_config (event);
@@ -2517,7 +2526,8 @@ static int intel_pmu_hw_config(struct perf_event *event)if (event->attr.precise_ip) {if (!event->attr.freq) { event->hw.flags |=PERF_X86_EVENT_AUTO_RELOAD ; -if (!(event->attr.sample_type & ~PEBS_FREERUNNING_FLAGS )) +if (!(event->attr.sample_type & + ~intel_pmu_free_running_flags (event))) event->hw.flags |=PERF_X86_EVENT_FREERUNNING ; }if (x86_pmu.pebs_aliases)
@@ -2954,6 +2964,8 @@ static __initconst const struct x86_pmu core_pmu = { .event_map = intel_pmu_event_map, .max_events = ARRAY_SIZE(intel_perfmon_event_map), .apic = 1, + .free_running_flags = PEBS_FREERUNNING_FLAGS, +/* * Intel PMCs cannot be accessed sanely above 32-bit width, * so we install an artificial 1<<31 period regardless of
@@ -2992,6 +3004,7 @@ static __initconst const struct x86_pmu intel_pmu = { .event_map = intel_pmu_event_map, .max_events = ARRAY_SIZE(intel_perfmon_event_map), .apic = 1, + .free_running_flags = PEBS_FREERUNNING_FLAGS,/* * Intel PMCs cannot be accessed sanely above 32 bit width, * so we install an artificial 1<<31 period regardless of
arch/x86/kernel/cpu/perf_event_intel_ds.c +1 -0
@@ -1295,6 +1295,7 @@ void __init intel_ds_init(void) x86_pmu.pebs_record_size =sizeof (struct pebs_record_skl ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm; + x86_pmu.free_running_flags |=PERF_SAMPLE_TIME ;break ;default :
Signature · 1 line
2.1.0
tip-bot for Andi Kleen
merge bot[tip:perf/core] perf/x86/intel/lbr: Allow time stamp for free running PEBSv3
perf/x86/intel/lbr: Allow time stamp for free running PEBSv3
Commit-ID: a7b58d211ba18c9175b139e18b68c86a6bcc3c3f Gitweb: http://git.kernel.org/tip/a7b58d211ba18c9175b139e18b68c86a6bcc3c3f Author: Andi Kleen <[email protected]> AuthorDate: Wed, 27 May 2015 21:13:14 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2015 10:16:56 +0200
perf/x86/intel/lbr: Allow time stamp for free running PEBSv3
With PEBSv3 the PEBS record contains a time stamp. That means we can allow free-running PEBS without a PMI even if the user program requested a time stamp. This avoids the need to use -T to get free running PEBS, and also avoids any problems with mis-identifying MMAPs later.
Move the free_running_flags state into a variable in x86_pmu and use it. This only works when no explicit clock_id is set.
--- arch/x86/kernel/cpu/perf_event.h | 1 + arch/x86/kernel/cpu/perf_event_intel.c | 15 ++++++++++++++- arch/x86/kernel/cpu/perf_event_intel_ds.c | 1 + 3 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/kernel/cpu/perf_event.h +1 -0
@@ -594,6 +594,7 @@ struct x86_pmu {struct event_constraint *pebs_constraints;void (*pebs_aliases)(struct perf_event *event);int max_pebs_events; +unsigned long free_running_flags;/* * Intel LBR
arch/x86/kernel/cpu/perf_event_intel.c +14 -1
@@ -2257,6 +2257,15 @@ static void intel_pebs_aliases_snb(struct perf_event *event) } } +static unsigned long intel_pmu_free_running_flags (struct perf_event *event) +{ +unsigned long flags = x86_pmu.free_running_flags; + +if (event->attr.use_clockid) + flags &= ~PERF_SAMPLE_TIME ; +return flags; +} +static int intel_pmu_hw_config (struct perf_event *event) {int ret =x86_pmu_hw_config (event);
@@ -2267,7 +2276,8 @@ static int intel_pmu_hw_config(struct perf_event *event)if (event->attr.precise_ip) {if (!event->attr.freq) { event->hw.flags |=PERF_X86_EVENT_AUTO_RELOAD ; -if (!(event->attr.sample_type & ~PEBS_FREERUNNING_FLAGS )) +if (!(event->attr.sample_type & + ~intel_pmu_free_running_flags (event))) event->hw.flags |=PERF_X86_EVENT_FREERUNNING ; }if (x86_pmu.pebs_aliases)
@@ -2689,6 +2699,8 @@ static __initconst const struct x86_pmu core_pmu = { .event_map = intel_pmu_event_map, .max_events = ARRAY_SIZE(intel_perfmon_event_map), .apic = 1, + .free_running_flags = PEBS_FREERUNNING_FLAGS, +/* * Intel PMCs cannot be accessed sanely above 32-bit width, * so we install an artificial 1<<31 period regardless of
@@ -2727,6 +2739,7 @@ static __initconst const struct x86_pmu intel_pmu = { .event_map = intel_pmu_event_map, .max_events = ARRAY_SIZE(intel_perfmon_event_map), .apic = 1, + .free_running_flags = PEBS_FREERUNNING_FLAGS,/* * Intel PMCs cannot be accessed sanely above 32 bit width, * so we install an artificial 1<<31 period regardless of
arch/x86/kernel/cpu/perf_event_intel_ds.c +1 -0
@@ -1273,6 +1273,7 @@ void __init intel_ds_init(void) x86_pmu.pebs_record_size =sizeof (struct pebs_record_skl ); x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm; + x86_pmu.free_running_flags |=PERF_SAMPLE_TIME ;break ;default :
PATCH 2/5 x86, perf: Add option to disable reading branch flags/cycles
In reply to Andi KleenFrom: Andi Kleen <[email protected]>
With LBRv5 reading the extra LBR flags like mispredict, TSX, cycles is not free anymore, as it has moved to a separate MSR.
For callstack mode we don't need any of this information; so we can avoid the unnecessary MSR read. Add flags to the perf interface where perf record can request not collecting this information. I added sample_type flags for CYCLES and FLAGS. It's a bit unusual for sample_types to be negative (disable), not positive (enable), but since the legacy ABI reported the flags we need some form of explicit disabling to avoid breaking the ABI. In theory it would be possible to make CYCLES opt-in (as it's not deployed yet), but I also made it opt-out to be symmetric to FLAGS.
After we have the flags the x86 perf code can keep track if any users need the flags. If noone needs it the information is not collected.
--- arch/x86/kernel/cpu/perf_event.h | 2 ++ arch/x86/kernel/cpu/perf_event_intel_lbr.c | 49 ++++++++++++++++++++++-------- include/uapi/linux/perf_event.h | 2 ++ 3 files changed, 40 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/perf_event.h +2 -0
@@ -201,6 +201,8 @@ struct cpu_hw_events {* Intel LBR bits */ int lbr_users; +int lbr_flags_users; +int lbr_cycles_users;void *lbr_context;struct perf_branch_stack lbr_stack;struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES ];
arch/x86/kernel/cpu/perf_event_intel_lbr.c +36 -13
87 lines not on this page · raw message
include/uapi/linux/perf_event.h +2 -0
@@ -138,6 +138,8 @@ enum perf_event_sample_format {PERF_SAMPLE_IDENTIFIER =1U <<16 ,PERF_SAMPLE_TRANSACTION =1U <<17 ,PERF_SAMPLE_REGS_INTR =1U <<18 , +PERF_SAMPLE_BRANCH_NO_FLAGS =1U <<19 , +PERF_SAMPLE_BRANCH_NO_CYCLES =1U <<20 , PERF_SAMPLE_MAX = 1U << 19, /* non-ABI */ };
Signature · 1 line
2.1.0
Peter Zijlstra
maintainerRe: [PATCH 2/5] x86, perf: Add option to disable reading branch flags/cycles
Quoted 24 lines · Andi Kleen
From: Andi Kleen <[email protected]>
With LBRv5 reading the extra LBR flags like mispredict, TSX, cycles is not free anymore, as it has moved to a separate MSR.
For callstack mode we don't need any of this information; so we can avoid the unnecessary MSR read. Add flags to the perf interface where perf record can request not collecting this information. I added
sample_typeflags for CYCLES and FLAGS. It's a bit unusual forsample_typesto be negative (disable), not positive (enable), but since the legacy ABI reported the flags we need some form of explicit disabling to avoid breaking the ABI. In theory it would be possible to make CYCLES opt-in (as it's not deployed yet), but I also made it opt-out to be symmetric to FLAGS.After we have the flags the x86 perf code can keep track if any users need the flags. If noone needs it the information is not collected.
arch/x86/kernel/cpu/perf_event_intel_lbr.c +0
-0
arch/x86/kernel/cpu/perf_event_intel_lbr.c void intel_pmu_lbr_enable(struct perf_event *event) · line 348
Quoted 9 lines
arch/x86/kernel/cpu/perf_event_intel_lbr.c +4
-0@@ -340,6 +340,10 @@ void intel_pmu_lbr_enable(struct perf_event *event) } cpuc->lbr_users++; +if (!(event->attr.sample_type &PERF_SAMPLE_BRANCH_NO_FLAGS )) + cpuc->lbr_flags_users++; +if (!(event->attr.sample_type &PERF_SAMPLE_BRANCH_NO_CYCLES )) + cpuc->lbr_cycles_users++;perf_sched_cb_inc (event->ctx->pmu); }
This patch seems to add an unfortunate amount of branches. And while I appreciate it'll all be cheaper than the SKL MSR read, it does add overhead to the older chips.
Also, I think its broken. We can now have two events, with the 'same' LBR config but with different flags/cycles settings. __intel_shared_reg_get_constraints() will find a match, but the enable code above will disable flags/cycles for both of them.
include/uapi/linux/perf_event.h enum perf_event_sample_format { · line 145
Quoted 13 lines
include/uapi/linux/perf_event.h +2
-0@@ -138,6 +138,8 @@ enum perf_event_sample_format {PERF_SAMPLE_IDENTIFIER =1U <<16 ,PERF_SAMPLE_TRANSACTION =1U <<17 ,PERF_SAMPLE_REGS_INTR =1U <<18 , +PERF_SAMPLE_BRANCH_NO_FLAGS =1U <<19 , +PERF_SAMPLE_BRANCH_NO_CYCLES =1U <<20 , PERF_SAMPLE_MAX = 1U << 19, /* non-ABI */ };
Should this not be part of perf_branch_sample_type instead? That seems to otherwise specify all the branch stack details.
PATCH 3/5 perf, tools: Disable branch flags/cycles for lbr call graph
In reply to Andi KleenFrom: Andi Kleen <[email protected]>
Automatically disable collecting branch flags and cycles with --call-graph lbr. This allows avoiding a bunch of extra MSR reads in the PMI on Skylake.
When the kernel doesn't support the new flags they are automatically cleared in the fallback code.
--- tools/perf/util/evsel.c | 4 ++++ 1 file changed, 4 insertions(+)
tools/perf/util/evsel.c +4 -0
@@ -557,6 +557,8 @@ perf_evsel__config_callgraph(struct perf_evsel *evsel, "Falling back to framepointers.\n"); }else {perf_evsel__set_sample_bit (evsel,BRANCH_STACK ); +perf_evsel__set_sample_bit (evsel,BRANCH_NO_CYCLES ); +perf_evsel__set_sample_bit (evsel,BRANCH_NO_FLAGS ); attr->branch_sample_type =PERF_SAMPLE_BRANCH_USER |PERF_SAMPLE_BRANCH_CALL_STACK ; }
@@ -1167,6 +1169,8 @@ fallback_missing_features: evsel->attr.mmap2 =0 ;if (perf_missing_features.exclude_guest) evsel->attr.exclude_guest = evsel->attr.exclude_host =0 ; + evsel->attr.sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS | +PERF_SAMPLE_BRANCH_NO_CYCLES ); retry_sample_id:if (perf_missing_features.sample_id_all) evsel->attr.sample_id_all =0 ;
Signature · 1 line
2.1.0
PATCH 4/5 x86, perf: Use correct index to save/restore LBR_INFO with callstack
In reply to Andi KleenFrom: Andi Kleen <[email protected]>
Use the correct index to save/restore the LBR_INFO_x MSR in callstack mode. This is more a cleanup, as even with the wrong index the register was correctly saved/restored, and also LBR callgraph mode in perf tools do not really need anything in LBR_INFO. But still better to use the right index.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +2 -2
@@ -243,7 +243,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)wrmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);wrmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_INFO ) -wrmsrl (MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]); +wrmsrl (MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } task_ctx->lbr_stack_state =LBR_NONE ; }
@@ -266,7 +266,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)rdmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);rdmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_INFO ) -rdmsrl (MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]); +rdmsrl (MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } task_ctx->lbr_stack_state =LBR_VALID ; }
Signature · 1 line
2.1.0
tip-bot for Andi Kleen
merge bot[tip:perf/core] perf/x86/intel/lbr: Use correct index to save/ restore LBR_INFO with call stack
perf/x86/intel/lbr: Use correct index to save/restore LBR_INFO with call stack
Commit-ID: e0573364b8c5b17401569ef581f1625803210f4d Gitweb: http://git.kernel.org/tip/e0573364b8c5b17401569ef581f1625803210f4d Author: Andi Kleen <[email protected]> AuthorDate: Wed, 27 May 2015 21:13:17 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2015 10:16:59 +0200
perf/x86/intel/lbr: Use correct index to save/restore LBR_INFO with call stack
Use the correct index to save/restore the LBR_INFO_x MSR in callstack mode. This is more a cleanup, as even with the wrong index the register was correctly saved/restored, and also LBR callgraph mode in perf tools do not really need anything in LBR_INFO. But still better to use the right index.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +2 -2
@@ -245,7 +245,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)wrmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);wrmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_INFO ) -wrmsrl (MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]); +wrmsrl (MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } task_ctx->lbr_stack_state =LBR_NONE ; }
@@ -268,7 +268,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)rdmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);rdmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);if (x86_pmu.intel_cap.lbr_format ==LBR_FORMAT_INFO ) -rdmsrl (MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]); +rdmsrl (MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } task_ctx->lbr_stack_state =LBR_VALID ; }
PATCH 5/5 x86, perf: Limit LBR accesses to TOS in callstack mode
In reply to Andi KleenFrom: Andi Kleen <[email protected]>
In callstack mode the LBR is not a ring buffer, but a stack that grows up and down. This means in this case we don't need to access all LBRs, only the ones up to TOS. Do this optimization for the normal LBR read, and the context switch save/restore code. For save/restore it can be done unconditionally, as it only runs when call stack mode is active.
This recovers some of the cost of going to 32 LBRs on Skylake.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +7 -3
@@ -238,7 +238,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) mask = x86_pmu.lbr_nr -1 ; tos =intel_pmu_lbr_tos (); -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +for (i =0 ; i < tos; i++) { lbr_idx = (tos - i) & mask;wrmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);wrmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -261,7 +261,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) mask = x86_pmu.lbr_nr -1 ; tos =intel_pmu_lbr_tos (); -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +for (i =0 ; i < tos; i++) { lbr_idx = (tos - i) & mask;rdmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);rdmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -436,8 +436,12 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,u64 tos =intel_pmu_lbr_tos ();int i;int out =0 ; +int num = x86_pmu.lbr_nr; -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +if (cpuc->lbr_sel->config &LBR_CALL_STACK ) + num = tos; + +for (i =0 ; i < num; i++) {unsigned long lbr_idx = (tos - i) & mask;u64 from, to, mis =0 , pred =0 , in_tx =0 , abort =0 ;int skip =0 ;
Signature · 1 line
2.1.0
tip-bot for Andi Kleen
merge bot[tip:perf/core] perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode
perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode
Commit-ID: 90405aa02247c1a6313c33e2253f9fd2299ae60b Gitweb: http://git.kernel.org/tip/90405aa02247c1a6313c33e2253f9fd2299ae60b Author: Andi Kleen <[email protected]> AuthorDate: Wed, 27 May 2015 21:13:18 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2015 10:16:59 +0200
perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode
In callstack mode the LBR is not a ring buffer, but a stack that grows up and down. This means in this case we don't need to access all LBRs, only the ones up to TOS. Do this optimization for the normal LBR read, and the context switch save/restore code. For save/restore it can be done unconditionally, as it only runs when call stack mode is active.
This recovers some of the cost of going to 32 LBRs on Skylake.
--- arch/x86/kernel/cpu/perf_event_intel_lbr.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +7 -3
@@ -240,7 +240,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) mask = x86_pmu.lbr_nr -1 ; tos =intel_pmu_lbr_tos (); -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +for (i =0 ; i < tos; i++) { lbr_idx = (tos - i) & mask;wrmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);wrmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -263,7 +263,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) mask = x86_pmu.lbr_nr -1 ; tos =intel_pmu_lbr_tos (); -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +for (i =0 ; i < tos; i++) { lbr_idx = (tos - i) & mask;rdmsrl (x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);rdmsrl (x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -425,8 +425,12 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)u64 tos =intel_pmu_lbr_tos ();int i;int out =0 ; +int num = x86_pmu.lbr_nr; -for (i =0 ; i < x86_pmu.lbr_nr; i++) { +if (cpuc->lbr_sel->config &LBR_CALL_STACK ) + num = tos; + +for (i =0 ; i < num; i++) {unsigned long lbr_idx = (tos - i) & mask;u64 from, to, mis =0 , pred =0 , in_tx =0 , abort =0 ;int skip =0 ;