Skip to contentSkip to conversation

perf/x86/amd: support capturing LBR from software events

PATCH In review Andrii Nakryiko 3 participants 4 messages Mar 29, 2024 – Mar 31, 2024

Subsystems (3)
lore
Share & tools
lore

Andrii Nakryiko

author maintainer
#0 lore raw

PATCH v3 1/2 perf/x86/amd: support capturing LBR from software events

[0] added ability to capture LBR (Last Branch Records) on Intel CPUs from inside BPF program at pretty much any arbitrary point. This is extremely useful capability that allows to figure out otherwise hard-to-debug problems, because LBR is now available based on some application-defined conditions, not just hardware-supported events.

retsnoop ([1]) is one such tool that takes a huge advantage of this functionality and has proved to be an extremely useful tool in practice.

Now, AMD Zen4 CPUs got support for similar LBR functionality, but necessary wiring inside the kernel is not yet setup. This patch seeks to rectify this and follows a similar approach to the original patch [0] for Intel CPUs.

Given LBR can be set up to capture any indirect jumps, it's critical to minimize indirect jumps on the way to requesting LBR from BPF program, so we split amd_pmu_lbr_disable_all() into a wrapper with some generic conditions vs always-inlined __amd_pmu_lbr_disable() called directly from BPF subsystem (through perf_snapshot_branch_stack static call).

This was tested on AMD Bergamo CPU and worked well when utilized from the aforementioned retsnoop tool.

[0] https://lore.kernel.org/bpf/[email protected]/
[1] https://github.com/anakryiko/retsnoop

---
 arch/x86/events/amd/core.c   | 29 ++++++++++++++++++++++++++++-
 arch/x86/events/amd/lbr.c    |  7 +------
 arch/x86/events/perf_event.h | 11 +++++++++++
 3 files changed, 40 insertions(+), 7 deletions(-)
arch/x86/events/amd/core.c +28 -1
index aec16e581f5b..88f6d0701342 100644
@@ -618,7 +618,7 @@ static void amd_pmu_cpu_dead(int cpu)
 	}
 }
 
-static inline void amd_pmu_set_global_ctl(u64 ctl)
+static __always_inline void amd_pmu_set_global_ctl(u64 ctl)
 {
 	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 }
@@ -878,6 +878,29 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
 	return amd_pmu_adjust_nmi_window(handled);
 }
 
+static int amd_pmu_v2_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)
+{
+	struct cpu_hw_events *cpuc;
+	unsigned long flags;
+
+	/* must not have branches... */
+	local_irq_save(flags);
+	amd_pmu_core_disable_all();
+	__amd_pmu_lbr_disable();
+	/*            ... until here */
+
+	cpuc = this_cpu_ptr(&cpu_hw_events);
+
+	amd_pmu_lbr_read();
+	cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr);
+	memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);
+
+	amd_pmu_v2_enable_all(0);
+	local_irq_restore(flags);
+
+	return cnt;
+}
+
 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1414,6 +1437,10 @@ static int __init amd_core_pmu_init(void)
 		static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
 		static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
 		static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
+
+		/* only support branch_stack snapshot on perfmon v2 */
+		if (x86_pmu.handle_irq == amd_pmu_v2_handle_irq)
+			static_call_update(perf_snapshot_branch_stack, amd_pmu_v2_snapshot_branch_stack);
 	} else if (!amd_brs_init()) {
 		/*
 		 * BRS requires special event constraints and flushing on ctxsw.
arch/x86/events/amd/lbr.c +1 -6
index 4a1e600314d5..0e4de028590d 100644
@@ -412,16 +412,11 @@ void amd_pmu_lbr_enable_all(void)
 void amd_pmu_lbr_disable_all(void)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	u64 dbg_ctl, dbg_extn_cfg;
 
 	if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
 		return;
 
-	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-	rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-
-	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
-	wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+	__amd_pmu_lbr_disable();
 }
 
 __init int amd_pmu_lbr_init(void)
arch/x86/events/perf_event.h +11 -0
index fb56518356ec..4dddf0a7e81e 100644
@@ -1329,6 +1329,17 @@ void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
 
+static __always_inline void __amd_pmu_lbr_disable(void)
+{
+	u64 dbg_ctl, dbg_extn_cfg;
+
+	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+	rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+
+	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
+	wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+}
+
 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
 
 #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
Signature · 1 line
2.43.0
author maintainer
#1 lore raw

PATCH v3 2/2 perf/x86/amd: don't reject sampling events with configured LBR

Now that it's possible to capture LBR on AMD CPU from BPF at arbitrary point, there is no reason to artificially limit this feature to sampling events. So corresponding check is removed. AFAIU, there is no correctness implications of doing this (and it was possible to bypass this check by just setting perf_event's sample_period to 1 anyways, so it doesn't guard all that much).

---
 arch/x86/events/amd/lbr.c | 4 ----
 1 file changed, 4 deletions(-)
arch/x86/events/amd/lbr.c +0 -4
index 0e4de028590d..75920f895d67 100644
@@ -310,10 +310,6 @@ int amd_pmu_lbr_hw_config(struct perf_event *event)
 {
 	int ret = 0;
 
-	/* LBR is not recommended in counting mode */
-	if (!is_sampling_event(event))
-		return -EINVAL;
-
 	ret = amd_pmu_lbr_setup_filter(event);
 	if (!ret)
 		event->attach_state |= PERF_ATTACH_SCHED_CB;
Signature · 1 line
2.43.0
#2 lore raw
In reply to Andrii Nakryiko
Quoted 2 lines · Andrii Nakryiko

[0] added ability to capture LBR (Last Branch Records) on Intel CPUs from inside BPF program at pretty much any arbitrary point.

Upstream commit ID:

c22ac2a3d4bd ("perf: Enable branch record for software events")

Quoted 39 lines

[...] This is extremely useful capability that allows to figure out otherwise hard-to-debug problems, because LBR is now available based on some application-defined conditions, not just hardware-supported events.

retsnoop ([1]) is one such tool that takes a huge advantage of this functionality and has proved to be an extremely useful tool in practice.

Now, AMD Zen4 CPUs got support for similar LBR functionality, but necessary wiring inside the kernel is not yet setup. This patch seeks to rectify this and follows a similar approach to the original patch [0] for Intel CPUs.

Given LBR can be set up to capture any indirect jumps, it's critical to minimize indirect jumps on the way to requesting LBR from BPF program, so we split amd_pmu_lbr_disable_all() into a wrapper with some generic conditions vs always-inlined __amd_pmu_lbr_disable() called directly from BPF subsystem (through perf_snapshot_branch_stack static call).

This was tested on AMD Bergamo CPU and worked well when utilized from the aforementioned retsnoop tool.

[0] https://lore.kernel.org/bpf/[email protected]/
[1] https://github.com/anakryiko/retsnoop

---
 arch/x86/events/amd/core.c   | 29 ++++++++++++++++++++++++++++-
 arch/x86/events/amd/lbr.c    |  7 +------
 arch/x86/events/perf_event.h | 11 +++++++++++
 3 files changed, 40 insertions(+), 7 deletions(-)
arch/x86/events/amd/core.c +0 -0
index aec16e581f5b..88f6d0701342 100644

arch/x86/events/amd/core.c static void amd_pmu_cpu_dead(int cpu) · line 621

Quoted 5 lines
arch/x86/events/amd/core.c +1 -1
@@ -618,7 +618,7 @@ static void amd_pmu_cpu_dead(int cpu)
 	}
 }
 
-static inline void amd_pmu_set_global_ctl(u64 ctl)
+static __always_inline void amd_pmu_set_global_ctl(u64 ctl)

What is this inlining change about? My first guess was that it's to generate better code, but my guess was wrong: it's to avoid branches. To not force people to guess, please put it into a separate patch & add an explanation.

Quoted 7 lines
arch/x86/events/amd/core.c +0 -0
 {
 	wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 }
@@ -878,6 +878,29 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
 	return amd_pmu_adjust_nmi_window(handled);
 }
 

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 890

Quoted 10 lines
arch/x86/events/amd/core.c +10 -0
+static int amd_pmu_v2_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)
+{
+	struct cpu_hw_events *cpuc;
+	unsigned long flags;
+
+	/* must not have branches... */
+	local_irq_save(flags);
+	amd_pmu_core_disable_all();
+	__amd_pmu_lbr_disable();
+	/*            ... until here */

Oh ... so it's not about performance or code layout, but to avoid new branches to contaminate the snapshot, right? Even stronger reason to put that change into a separate patch.

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 895

Quoted 5 lines
arch/x86/events/amd/core.c +5 -0
+
+	cpuc = this_cpu_ptr(&cpu_hw_events);
+
+	amd_pmu_lbr_read();
+	cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr);

Why is min_t() used here? AFAICT all types here are 'unsigned int'.

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 896

Quoted 1 line
arch/x86/events/amd/core.c +1 -0
+	memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);

The function could use a description comment explaining the arguments, and that the caller must make sure there's enough space in the 'entries' array.

Quoted 21 lines
arch/x86/events/amd/core.c +11 -0
+
+	amd_pmu_v2_enable_all(0);
+	local_irq_restore(flags);
+
+	return cnt;
+}
+
 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1414,6 +1437,10 @@ static int __init amd_core_pmu_init(void)
 		static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
 		static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
 		static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
+
+		/* only support branch_stack snapshot on perfmon v2 */
+		if (x86_pmu.handle_irq == amd_pmu_v2_handle_irq)
+			static_call_update(perf_snapshot_branch_stack, amd_pmu_v2_snapshot_branch_stack);
 
 	} else if (!amd_brs_init()) {
 		/*
Quoted 1 line

* BRS requires special event constraints and flushing on ctxsw.

Please use consistent capitalization in all new comments you add:

	/* Properly capitalized comment */
Quoted 31 lines
arch/x86/events/amd/lbr.c +1 -6
index 4a1e600314d5..0e4de028590d 100644
@@ -412,16 +412,11 @@ void amd_pmu_lbr_enable_all(void)
 void amd_pmu_lbr_disable_all(void)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	u64 dbg_ctl, dbg_extn_cfg;
 
 	if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
 		return;
 
-	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-	rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-
-	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
-	wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+	__amd_pmu_lbr_disable();
 }
 
 __init int amd_pmu_lbr_init(void)
arch/x86/events/perf_event.h +1 -0
index fb56518356ec..4dddf0a7e81e 100644
@@ -1329,6 +1329,17 @@ void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
 
+static __always_inline void __amd_pmu_lbr_disable(void)

arch/x86/events/perf_event.h void amd_pmu_lbr_enable_all(void); · line 1342

Quoted 10 lines
arch/x86/events/perf_event.h +10 -0
+{
+	u64 dbg_ctl, dbg_extn_cfg;
+
+	rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+	rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+
+	wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
+	wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+}
+

This factoring out of __amd_pmu_lbr_disable() should be in a separate preparatory patch too.

Thanks,

Ingo

#3 lore raw
Quoted 10 lines · Ingo Molnar
Quoted 2 lines · Andrii Nakryiko

[0] added ability to capture LBR (Last Branch Records) on Intel CPUs from inside BPF program at pretty much any arbitrary point.

Upstream commit ID:

c22ac2a3d4bd ("perf: Enable branch record for software events")

will include this in commit message

Quoted 43 lines
Quoted 43 lines

[...] This is extremely useful capability that allows to figure out otherwise hard-to-debug problems, because LBR is now available based on some application-defined conditions, not just hardware-supported events.

retsnoop ([1]) is one such tool that takes a huge advantage of this functionality and has proved to be an extremely useful tool in practice.

Now, AMD Zen4 CPUs got support for similar LBR functionality, but necessary wiring inside the kernel is not yet setup. This patch seeks to rectify this and follows a similar approach to the original patch [0] for Intel CPUs.

Given LBR can be set up to capture any indirect jumps, it's critical to minimize indirect jumps on the way to requesting LBR from BPF program, so we split amd_pmu_lbr_disable_all() into a wrapper with some generic conditions vs always-inlined __amd_pmu_lbr_disable() called directly from BPF subsystem (through perf_snapshot_branch_stack static call).

This was tested on AMD Bergamo CPU and worked well when utilized from the aforementioned retsnoop tool.

[0] https://lore.kernel.org/bpf/[email protected]/
[1] https://github.com/anakryiko/retsnoop

---
 arch/x86/events/amd/core.c   | 29 ++++++++++++++++++++++++++++-
 arch/x86/events/amd/lbr.c    |  7 +------
 arch/x86/events/perf_event.h | 11 +++++++++++
 3 files changed, 40 insertions(+), 7 deletions(-)
arch/x86/events/amd/core.c +0 -0
index aec16e581f5b..88f6d0701342 100644
@@ -618,7 +618,7 @@ static void amd_pmu_cpu_dead(int cpu)
      }
 }
 
Quoted 7 lines

arch/x86/events/amd/core.c static void amd_pmu_cpu_dead(int cpu) · line 621

Quoted 2 lines
arch/x86/events/amd/core.c +1 -1
-static inline void amd_pmu_set_global_ctl(u64 ctl)
+static __always_inline void amd_pmu_set_global_ctl(u64 ctl)

What is this inlining change about? My first guess was that it's to generate better code, but my guess was wrong: it's to avoid branches. To not force people to guess, please put it into a separate patch & add an explanation.

ok, will do

Quoted 8 lines
Quoted 8 lines
arch/x86/events/amd/core.c +0 -0
 {
      wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 }
@@ -878,6 +878,29 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
      return amd_pmu_adjust_nmi_window(handled);
 }
 
Quoted 15 lines

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 890

Quoted 10 lines
arch/x86/events/amd/core.c +10 -0
+static int amd_pmu_v2_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)
+{
+     struct cpu_hw_events *cpuc;
+     unsigned long flags;
+
+     /* must not have branches... */
+     local_irq_save(flags);
+     amd_pmu_core_disable_all();
+     __amd_pmu_lbr_disable();
+     /*            ... until here */

Oh ... so it's not about performance or code layout, but to avoid new branches to contaminate the snapshot, right? Even stronger reason to put that change into a separate patch.

yep

Quoted 7 lines

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 895

Quoted 5 lines
arch/x86/events/amd/core.c +5 -0
+
+     cpuc = this_cpu_ptr(&cpu_hw_events);
+
+     amd_pmu_lbr_read();
+     cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr);

Why is min_t() used here? AFAICT all types here are 'unsigned int'.

you are right, seems like it's all unsigned int, so I will drop min_t. Not sure why the original Intel implementation used min_t().

Quoted 7 lines

arch/x86/events/amd/core.c static int amd_pmu_handle_irq(struct pt_regs *regs) · line 896

Quoted 2 lines
arch/x86/events/amd/core.c +1 -0
+     memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);

The function could use a description comment explaining the arguments, and that the caller must make sure there's enough space in the 'entries' array.

There is a comment next to perf_snapshot_branch_stack static call definition in include/linux/perf_event.h. I'll include a comment that amd_pmu_v2_snapshot_branch_stack() is an AMD-specific "implementation" of that static call, so user can refer to it for documentation (which would be kept in one place this way, instead of copy/pasting it between Intel and AMD implementations).

Quoted 19 lines
Quoted 18 lines
arch/x86/events/amd/core.c +11 -0
+
+     amd_pmu_v2_enable_all(0);
+     local_irq_restore(flags);
+
+     return cnt;
+}
+
 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 {
      struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1414,6 +1437,10 @@ static int __init amd_core_pmu_init(void)
              static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
              static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
              static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
+
+             /* only support branch_stack snapshot on perfmon v2 */
+             if (x86_pmu.handle_irq == amd_pmu_v2_handle_irq)
+                     static_call_update(perf_snapshot_branch_stack, amd_pmu_v2_snapshot_branch_stack);
Quoted 8 lines

arch/x86/events/amd/core.c static int __init amd_core_pmu_init(void) · line 1446

Quoted 3 lines
arch/x86/events/amd/core.c +0 -0
      } else if (!amd_brs_init()) {
              /*
               * BRS requires special event constraints and flushing on ctxsw.

Please use consistent capitalization in all new comments you add:

        /* Properly capitalized comment */

ack

Quoted 31 lines
Quoted 31 lines
arch/x86/events/amd/lbr.c +1 -6
index 4a1e600314d5..0e4de028590d 100644
@@ -412,16 +412,11 @@ void amd_pmu_lbr_enable_all(void)
 void amd_pmu_lbr_disable_all(void)
 {
      struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-     u64 dbg_ctl, dbg_extn_cfg;
 
      if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
              return;
 
-     rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-     rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-
-     wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
-     wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+     __amd_pmu_lbr_disable();
 }
 
 __init int amd_pmu_lbr_init(void)
arch/x86/events/perf_event.h +0 -0
index fb56518356ec..4dddf0a7e81e 100644
@@ -1329,6 +1329,17 @@ void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
 
Quoted 15 lines

arch/x86/events/perf_event.h void amd_pmu_lbr_enable_all(void); · line 1342

Quoted 11 lines
arch/x86/events/perf_event.h +11 -0
+static __always_inline void __amd_pmu_lbr_disable(void)
+{
+     u64 dbg_ctl, dbg_extn_cfg;
+
+     rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+     rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+
+     wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
+     wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+}
+

This factoring out of __amd_pmu_lbr_disable() should be in a separate preparatory patch too.

ok

Quoted 3 lines

Thanks,

Ingo