Skip to contentSkip to conversation

PATCH In review Andi Kleen 3 participants 10 messages May 28, 2015 – Aug 4, 2015

Subsystems (2)
lore
Share & tools
lore

Andi Kleen

author
#0 lore raw

[Repost; I forgot to copy linux-kernel earlier. Apologies if you see it twice.]

Skylake moved to 32 Last Branch Records, from previously 16. The current call stack LBR implementation reads all LBRs and also saves/restores them on context switch. This patchkit adds some optimizations to avoid extra costs in most cases from the larger number of LBRs for call-stack, unless a nesting larger than 16 is actually needed. It applies on top of the earlier Skylake code. Some of the optimization will also benefit earlier CPUs, such as Haswell.

note: one patch is for perf user space, the rest is kernel.

-Andi

author
#1 lore raw

PATCH 1/5 x86, perf: Allow time stamp for free running PEBSv3

From: Andi Kleen <[email protected]>

With PEBSv3 the PEBS record contains a time stamp. That means we can allow free-running PEBS without a PMI even if the user program requested a time stamp. This avoids the need to use -T to get free running PEBS, and also avoids any problems with mis-identifying MMAPs later.

Move the free_running_flags state into a variable in x86_pmu and use it. This only works when no explicit clock_id is set.

---
 arch/x86/kernel/cpu/perf_event.h          |  1 +
 arch/x86/kernel/cpu/perf_event_intel.c    | 15 ++++++++++++++-
 arch/x86/kernel/cpu/perf_event_intel_ds.c |  1 +
 3 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/kernel/cpu/perf_event.h +1 -0
index d1388fe..2860b89 100644
@@ -589,6 +589,7 @@ struct x86_pmu {
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	int 		max_pebs_events;
+	unsigned long	free_running_flags;
 
 	/*
 	 * Intel LBR
arch/x86/kernel/cpu/perf_event_intel.c +14 -1
index 626e7db..7ab1ba1 100644
@@ -2507,6 +2507,15 @@ static void intel_pebs_aliases_snb(struct perf_event *event)
 	}
 }
 
+static unsigned long intel_pmu_free_running_flags(struct perf_event *event)
+{
+	unsigned long flags = x86_pmu.free_running_flags;
+
+	if (event->attr.clockid)
+		flags &= ~PERF_SAMPLE_TIME;
+	return flags;
+}
+
 static int intel_pmu_hw_config(struct perf_event *event)
 {
 	int ret = x86_pmu_hw_config(event);
@@ -2517,7 +2526,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	if (event->attr.precise_ip) {
 		if (!event->attr.freq) {
 			event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
-			if (!(event->attr.sample_type & ~PEBS_FREERUNNING_FLAGS))
+			if (!(event->attr.sample_type &
+			      ~intel_pmu_free_running_flags(event)))
 				event->hw.flags |= PERF_X86_EVENT_FREERUNNING;
 		}
 		if (x86_pmu.pebs_aliases)
@@ -2954,6 +2964,8 @@ static __initconst const struct x86_pmu core_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
+	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
+
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32-bit width,
 	 * so we install an artificial 1<<31 period regardless of
@@ -2992,6 +3004,7 @@ static __initconst const struct x86_pmu intel_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
+	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32 bit width,
 	 * so we install an artificial 1<<31 period regardless of
arch/x86/kernel/cpu/perf_event_intel_ds.c +1 -0
index f7f28c3..22f236b 100644
@@ -1295,6 +1295,7 @@ void __init intel_ds_init(void)
 			x86_pmu.pebs_record_size =
 						sizeof(struct pebs_record_skl);
 			x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
+			x86_pmu.free_running_flags |= PERF_SAMPLE_TIME;
 			break;
 
 		default:
Signature · 1 line
2.1.0
#2 lore raw

[tip:perf/core] perf/x86/intel/lbr: Allow time stamp for free running PEBSv3

perf/x86/intel/lbr: Allow time stamp for free running PEBSv3
Commit-ID:  a7b58d211ba18c9175b139e18b68c86a6bcc3c3f
Gitweb:     http://git.kernel.org/tip/a7b58d211ba18c9175b139e18b68c86a6bcc3c3f
Author:     Andi Kleen <[email protected]>
AuthorDate: Wed, 27 May 2015 21:13:14 -0700
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 4 Aug 2015 10:16:56 +0200

perf/x86/intel/lbr: Allow time stamp for free running PEBSv3

With PEBSv3 the PEBS record contains a time stamp. That means we can allow free-running PEBS without a PMI even if the user program requested a time stamp. This avoids the need to use -T to get free running PEBS, and also avoids any problems with mis-identifying MMAPs later.

Move the free_running_flags state into a variable in x86_pmu and use it. This only works when no explicit clock_id is set.

---
 arch/x86/kernel/cpu/perf_event.h          |  1 +
 arch/x86/kernel/cpu/perf_event_intel.c    | 15 ++++++++++++++-
 arch/x86/kernel/cpu/perf_event_intel_ds.c |  1 +
 3 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/kernel/cpu/perf_event.h +1 -0
index 3474cf2..7378b10 100644
@@ -594,6 +594,7 @@ struct x86_pmu {
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	int 		max_pebs_events;
+	unsigned long	free_running_flags;
 
 	/*
 	 * Intel LBR
arch/x86/kernel/cpu/perf_event_intel.c +14 -1
index 71815cf..cb112bf 100644
@@ -2257,6 +2257,15 @@ static void intel_pebs_aliases_snb(struct perf_event *event)
 	}
 }
 
+static unsigned long intel_pmu_free_running_flags(struct perf_event *event)
+{
+	unsigned long flags = x86_pmu.free_running_flags;
+
+	if (event->attr.use_clockid)
+		flags &= ~PERF_SAMPLE_TIME;
+	return flags;
+}
+
 static int intel_pmu_hw_config(struct perf_event *event)
 {
 	int ret = x86_pmu_hw_config(event);
@@ -2267,7 +2276,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	if (event->attr.precise_ip) {
 		if (!event->attr.freq) {
 			event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
-			if (!(event->attr.sample_type & ~PEBS_FREERUNNING_FLAGS))
+			if (!(event->attr.sample_type &
+			      ~intel_pmu_free_running_flags(event)))
 				event->hw.flags |= PERF_X86_EVENT_FREERUNNING;
 		}
 		if (x86_pmu.pebs_aliases)
@@ -2689,6 +2699,8 @@ static __initconst const struct x86_pmu core_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
+	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
+
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32-bit width,
 	 * so we install an artificial 1<<31 period regardless of
@@ -2727,6 +2739,7 @@ static __initconst const struct x86_pmu intel_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
+	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32 bit width,
 	 * so we install an artificial 1<<31 period regardless of
arch/x86/kernel/cpu/perf_event_intel_ds.c +1 -0
index 410270a..03773c2 100644
@@ -1273,6 +1273,7 @@ void __init intel_ds_init(void)
 			x86_pmu.pebs_record_size =
 						sizeof(struct pebs_record_skl);
 			x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
+			x86_pmu.free_running_flags |= PERF_SAMPLE_TIME;
 			break;
 
 		default:
author
#3 lore raw

PATCH 2/5 x86, perf: Add option to disable reading branch flags/cycles

In reply to Andi Kleen

From: Andi Kleen <[email protected]>

With LBRv5 reading the extra LBR flags like mispredict, TSX, cycles is not free anymore, as it has moved to a separate MSR.

For callstack mode we don't need any of this information; so we can avoid the unnecessary MSR read. Add flags to the perf interface where perf record can request not collecting this information. I added sample_type flags for CYCLES and FLAGS. It's a bit unusual for sample_types to be negative (disable), not positive (enable), but since the legacy ABI reported the flags we need some form of explicit disabling to avoid breaking the ABI. In theory it would be possible to make CYCLES opt-in (as it's not deployed yet), but I also made it opt-out to be symmetric to FLAGS.

After we have the flags the x86 perf code can keep track if any users need the flags. If noone needs it the information is not collected.

---
 arch/x86/kernel/cpu/perf_event.h           |  2 ++
 arch/x86/kernel/cpu/perf_event_intel_lbr.c | 49 ++++++++++++++++++++++--------
 include/uapi/linux/perf_event.h            |  2 ++
 3 files changed, 40 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/perf_event.h +2 -0
index 2860b89..c83bf07 100644
@@ -201,6 +201,8 @@ struct cpu_hw_events {
 	 * Intel LBR bits
 	 */
 	int				lbr_users;
+	int				lbr_flags_users;
+	int				lbr_cycles_users;
 	void				*lbr_context;
 	struct perf_branch_stack	lbr_stack;
 	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];
arch/x86/kernel/cpu/perf_event_intel_lbr.c +36 -13

87 lines not on this page · raw message

include/uapi/linux/perf_event.h +2 -0
index 1b8bd4a..8dd5765 100644
@@ -138,6 +138,8 @@ enum perf_event_sample_format {
 	PERF_SAMPLE_IDENTIFIER			= 1U << 16,
 	PERF_SAMPLE_TRANSACTION			= 1U << 17,
 	PERF_SAMPLE_REGS_INTR			= 1U << 18,
+	PERF_SAMPLE_BRANCH_NO_FLAGS		= 1U << 19,
+	PERF_SAMPLE_BRANCH_NO_CYCLES		= 1U << 20,
 
 	PERF_SAMPLE_MAX = 1U << 19,		/* non-ABI */
 };
Signature · 1 line
2.1.0
maintainer
#4 lore raw

Re: [PATCH 2/5] x86, perf: Add option to disable reading branch flags/cycles

Quoted 24 lines · Andi Kleen

From: Andi Kleen <[email protected]>

With LBRv5 reading the extra LBR flags like mispredict, TSX, cycles is not free anymore, as it has moved to a separate MSR.

For callstack mode we don't need any of this information; so we can avoid the unnecessary MSR read. Add flags to the perf interface where perf record can request not collecting this information. I added sample_type flags for CYCLES and FLAGS. It's a bit unusual for sample_types to be negative (disable), not positive (enable), but since the legacy ABI reported the flags we need some form of explicit disabling to avoid breaking the ABI. In theory it would be possible to make CYCLES opt-in (as it's not deployed yet), but I also made it opt-out to be symmetric to FLAGS.

After we have the flags the x86 perf code can keep track if any users need the flags. If noone needs it the information is not collected.

arch/x86/kernel/cpu/perf_event_intel_lbr.c +0 -0
index 1fd8b5a..5de2048 100644

arch/x86/kernel/cpu/perf_event_intel_lbr.c void intel_pmu_lbr_enable(struct perf_event *event) · line 348

Quoted 9 lines
arch/x86/kernel/cpu/perf_event_intel_lbr.c +4 -0
@@ -340,6 +340,10 @@ void intel_pmu_lbr_enable(struct perf_event *event)
 	}
 
 	cpuc->lbr_users++;
+	if (!(event->attr.sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS))
+		cpuc->lbr_flags_users++;
+	if (!(event->attr.sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES))
+		cpuc->lbr_cycles_users++;
 	perf_sched_cb_inc(event->ctx->pmu);
 }

This patch seems to add an unfortunate amount of branches. And while I appreciate it'll all be cheaper than the SKL MSR read, it does add overhead to the older chips.

Also, I think its broken. We can now have two events, with the 'same' LBR config but with different flags/cycles settings. __intel_shared_reg_get_constraints() will find a match, but the enable code above will disable flags/cycles for both of them.

include/uapi/linux/perf_event.h enum perf_event_sample_format { · line 145

Quoted 13 lines
include/uapi/linux/perf_event.h +2 -0
index 1b8bd4a..8dd5765 100644
@@ -138,6 +138,8 @@ enum perf_event_sample_format {
 	PERF_SAMPLE_IDENTIFIER			= 1U << 16,
 	PERF_SAMPLE_TRANSACTION			= 1U << 17,
 	PERF_SAMPLE_REGS_INTR			= 1U << 18,
+	PERF_SAMPLE_BRANCH_NO_FLAGS		= 1U << 19,
+	PERF_SAMPLE_BRANCH_NO_CYCLES		= 1U << 20,
 
 	PERF_SAMPLE_MAX = 1U << 19,		/* non-ABI */
 };

Should this not be part of perf_branch_sample_type instead? That seems to otherwise specify all the branch stack details.

author
#5 lore raw

PATCH 3/5 perf, tools: Disable branch flags/cycles for lbr call graph

In reply to Andi Kleen

From: Andi Kleen <[email protected]>

Automatically disable collecting branch flags and cycles with --call-graph lbr. This allows avoiding a bunch of extra MSR reads in the PMI on Skylake.

When the kernel doesn't support the new flags they are automatically cleared in the fallback code.

---
 tools/perf/util/evsel.c | 4 ++++
 1 file changed, 4 insertions(+)
tools/perf/util/evsel.c +4 -0
index c886b9f..4d8a86c 100644
@@ -557,6 +557,8 @@ perf_evsel__config_callgraph(struct perf_evsel *evsel,
 					   "Falling back to framepointers.\n");
 			} else {
 				perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
+				perf_evsel__set_sample_bit(evsel, BRANCH_NO_CYCLES);
+				perf_evsel__set_sample_bit(evsel, BRANCH_NO_FLAGS);
 				attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
 							PERF_SAMPLE_BRANCH_CALL_STACK;
 			}
@@ -1167,6 +1169,8 @@ fallback_missing_features:
 		evsel->attr.mmap2 = 0;
 	if (perf_missing_features.exclude_guest)
 		evsel->attr.exclude_guest = evsel->attr.exclude_host = 0;
+	evsel->attr.sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS|
+				     PERF_SAMPLE_BRANCH_NO_CYCLES);
 retry_sample_id:
 	if (perf_missing_features.sample_id_all)
 		evsel->attr.sample_id_all = 0;
Signature · 1 line
2.1.0
author
#6 lore raw

PATCH 4/5 x86, perf: Use correct index to save/restore LBR_INFO with callstack

In reply to Andi Kleen

From: Andi Kleen <[email protected]>

Use the correct index to save/restore the LBR_INFO_x MSR in callstack mode. This is more a cleanup, as even with the wrong index the register was correctly saved/restored, and also LBR callgraph mode in perf tools do not really need anything in LBR_INFO. But still better to use the right index.

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +2 -2
index 5de2048..fa5c731 100644
@@ -243,7 +243,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
 		wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
 		if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
-			wrmsrl(MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]);
+			wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
 	}
 	task_ctx->lbr_stack_state = LBR_NONE;
 }
@@ -266,7 +266,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
 		if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
-			rdmsrl(MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]);
+			rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
 	}
 	task_ctx->lbr_stack_state = LBR_VALID;
 }
Signature · 1 line
2.1.0
#7 lore raw

[tip:perf/core] perf/x86/intel/lbr: Use correct index to save/ restore LBR_INFO with call stack

perf/x86/intel/lbr: Use correct index to save/restore LBR_INFO with call stack
Commit-ID:  e0573364b8c5b17401569ef581f1625803210f4d
Gitweb:     http://git.kernel.org/tip/e0573364b8c5b17401569ef581f1625803210f4d
Author:     Andi Kleen <[email protected]>
AuthorDate: Wed, 27 May 2015 21:13:17 -0700
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 4 Aug 2015 10:16:59 +0200

perf/x86/intel/lbr: Use correct index to save/restore LBR_INFO with call stack

Use the correct index to save/restore the LBR_INFO_x MSR in callstack mode. This is more a cleanup, as even with the wrong index the register was correctly saved/restored, and also LBR callgraph mode in perf tools do not really need anything in LBR_INFO. But still better to use the right index.

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +2 -2
index b432c47..a5bc424 100644
@@ -245,7 +245,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
 		wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
 		if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
-			wrmsrl(MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]);
+			wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
 	}
 	task_ctx->lbr_stack_state = LBR_NONE;
 }
@@ -268,7 +268,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
 		if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
-			rdmsrl(MSR_LBR_INFO_0 + i, task_ctx->lbr_info[i]);
+			rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
 	}
 	task_ctx->lbr_stack_state = LBR_VALID;
 }
author
#8 lore raw

PATCH 5/5 x86, perf: Limit LBR accesses to TOS in callstack mode

In reply to Andi Kleen

From: Andi Kleen <[email protected]>

In callstack mode the LBR is not a ring buffer, but a stack that grows up and down. This means in this case we don't need to access all LBRs, only the ones up to TOS. Do this optimization for the normal LBR read, and the context switch save/restore code. For save/restore it can be done unconditionally, as it only runs when call stack mode is active.

This recovers some of the cost of going to 32 LBRs on Skylake.

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +7 -3
index fa5c731..8e1740e 100644
@@ -238,7 +238,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
 
 	mask = x86_pmu.lbr_nr - 1;
 	tos = intel_pmu_lbr_tos();
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	for (i = 0; i < tos; i++) {
 		lbr_idx = (tos - i) & mask;
 		wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -261,7 +261,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 
 	mask = x86_pmu.lbr_nr - 1;
 	tos = intel_pmu_lbr_tos();
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	for (i = 0; i < tos; i++) {
 		lbr_idx = (tos - i) & mask;
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -436,8 +436,12 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,
 	u64 tos = intel_pmu_lbr_tos();
 	int i;
 	int out = 0;
+	int num = x86_pmu.lbr_nr;
 
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	if (cpuc->lbr_sel->config & LBR_CALL_STACK)
+		num = tos;
+
+	for (i = 0; i < num; i++) {
 		unsigned long lbr_idx = (tos - i) & mask;
 		u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
 		int skip = 0;
Signature · 1 line
2.1.0
#9 lore raw

[tip:perf/core] perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode

perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode
Commit-ID:  90405aa02247c1a6313c33e2253f9fd2299ae60b
Gitweb:     http://git.kernel.org/tip/90405aa02247c1a6313c33e2253f9fd2299ae60b
Author:     Andi Kleen <[email protected]>
AuthorDate: Wed, 27 May 2015 21:13:18 -0700
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 4 Aug 2015 10:16:59 +0200

perf/x86/intel/lbr: Limit LBR accesses to TOS in callstack mode

In callstack mode the LBR is not a ring buffer, but a stack that grows up and down. This means in this case we don't need to access all LBRs, only the ones up to TOS. Do this optimization for the normal LBR read, and the context switch save/restore code. For save/restore it can be done unconditionally, as it only runs when call stack mode is active.

This recovers some of the cost of going to 32 LBRs on Skylake.

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +7 -3
index a5bc424..b2c9475 100644
@@ -240,7 +240,7 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
 
 	mask = x86_pmu.lbr_nr - 1;
 	tos = intel_pmu_lbr_tos();
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	for (i = 0; i < tos; i++) {
 		lbr_idx = (tos - i) & mask;
 		wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -263,7 +263,7 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 
 	mask = x86_pmu.lbr_nr - 1;
 	tos = intel_pmu_lbr_tos();
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	for (i = 0; i < tos; i++) {
 		lbr_idx = (tos - i) & mask;
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
 		rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
@@ -425,8 +425,12 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 	u64 tos = intel_pmu_lbr_tos();
 	int i;
 	int out = 0;
+	int num = x86_pmu.lbr_nr;
 
-	for (i = 0; i < x86_pmu.lbr_nr; i++) {
+	if (cpuc->lbr_sel->config & LBR_CALL_STACK)
+		num = tos;
+
+	for (i = 0; i < num; i++) {
 		unsigned long lbr_idx = (tos - i) & mask;
 		u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
 		int skip = 0;