perf/x86/intel: Don't clear perf metrics overflow bit unconditionally
Subsystems (3)
PATCH 1/2 perf/x86/intel: Don't clear perf metrics overflow bit unconditionally
The below code would always unconditionally clear other status bits like perf metrics overflow bit once PEBS buffer overflows.
status &= intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
This is incorrect. Perf metrics overflow bit should be cleared only when fixed counter 3 in PEBS counter group. Otherwise perf metrics overflow could be missed to handle.
--- arch/x86/events/intel/core.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
arch/x86/events/intel/core.c +11 -2
@@ -3140,7 +3140,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int bit;int handled =0 ; -u64 intel_ctrl =hybrid (cpuc->pmu, intel_ctrl);inc_irq_stat (apic_perf_irqs);
@@ -3184,7 +3183,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data);static_call (x86_pmu_drain_pebs)(regs, &data); - status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/* * PMI throttle may be triggered, which stops the PEBS event.
@@ -3195,6 +3193,15 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)*/ if (pebs_enabled != cpuc->pebs_enabled)wrmsrl (MSR_IA32_PEBS_ENABLE , cpuc->pebs_enabled); + +/* +* Above PEBS handler (PEBS counters snapshotting) has updated fixed +* counter 3 and perf metrics counts if they are in counter group, +* unnecessary to update again. +*/ +if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ] && +is_pebs_counter_event_group (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ])) + status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT ; }/*
@@ -3214,6 +3221,8 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)static_call (intel_pmu_update_topdown_event)(NULL ,NULL ); } + status &=hybrid (cpuc->pmu, intel_ctrl); +/* * Checkpointed counters can lead to 'spurious' PMIs because the * rollback caused by the PMI will have cleared the overflow status
base-commit: 5c3627b6f0595f1ec27e6f5df903bd072e9b9136
Signature · 1 line
2.40.1
PATCH 2/2 perf/x86/intel: Allow to update user space GPRs from PEBS records
Currently when user samples user space GPRs (--user-regs option) with PEBS, the user space GPRs actually always comes from software PMI instead of from PEBS hardware. This leads to the sampled GPRs may be inaccurate for single PEBS record case because of the skid between counter overflow and GPRs sampling on PMI.
For the large PEBS case, the thing is even worse. If user sets exclude_kernel attribute, large PEBS would be used to sample user space GPRs, but since PEBS GPRs group is not really enabled, it leads to all samples in the large PEBS record shares a same piece of user space GPRs, like below command shows.
perf record -e branches:pu --user-regs=ip,ax -c 100000 ./foo perf report -D | grep "AX"
.... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead
So enable GPRs group for user space GPRs sampling and prioritize reading GPRs from PEBS. If the PEBS sampled GPRs is not user space GPRs (single PEBS record case), perf_sample_regs_user() modifies them to user space GPRs.
--- arch/x86/events/intel/ds.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
arch/x86/events/intel/ds.c +5 -3
@@ -1398,8 +1398,10 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)* + precise_ip < 2 for the non event IP * + For RTM TSX weight we need GPRs for the abort code. */ - gprs = (sample_type &PERF_SAMPLE_REGS_INTR ) && - (attr->sample_regs_intr &PEBS_GP_REGS ); + gprs = ((sample_type &PERF_SAMPLE_REGS_INTR ) && + (attr->sample_regs_intr &PEBS_GP_REGS )) || + ((sample_type &PERF_SAMPLE_REGS_USER ) && + (attr->sample_regs_user &PEBS_GP_REGS )); tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) && ((attr->config & INTEL_ARCH_EVENT_MASK) ==
@@ -2122,7 +2124,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, regs->flags &= ~PERF_EFLAGS_EXACT ; } -if (sample_type &PERF_SAMPLE_REGS_INTR ) +if (sample_type & (PERF_SAMPLE_REGS_INTR |PERF_SAMPLE_REGS_USER ))adaptive_pebs_save_regs (regs, gprs); }
Signature · 1 line
2.40.1
[tip: perf/core] perf/x86/intel: Allow to update user space GPRs from PEBS records
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 71dcc11c2cd9e434c34a63154ecadca21c135ddd Gitweb: https://git.kernel.org/tip/71dcc11c2cd9e434c34a63154ecadca21c135ddd Author: Dapeng Mi <[email protected]> AuthorDate: Tue, 15 Apr 2025 10:41:35 Committer: Ingo Molnar <[email protected]> CommitterDate: Thu, 17 Apr 2025 14:19:38 +02:00
perf/x86/intel: Allow to update user space GPRs from PEBS records
Currently when a user samples user space GPRs (--user-regs option) with PEBS, the user space GPRs actually always come from software PMI instead of from PEBS hardware. This leads to the sampled GPRs to possibly be inaccurate for single PEBS record case because of the skid between counter overflow and GPRs sampling on PMI.
For the large PEBS case, it is even worse. If user sets the exclude_kernel attribute, large PEBS would be used to sample user space GPRs, but since PEBS GPRs group is not really enabled, it leads to all samples in the large PEBS record to share the same piece of user space GPRs, like this reproducer shows:
$ perf record -e branches:pu --user-regs=ip,ax -c 100000 ./foo $ perf report -D | grep "AX"
.... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead .... AX 0x000000003a0d4ead
So enable GPRs group for user space GPRs sampling and prioritize reading GPRs from PEBS. If the PEBS sampled GPRs is not user space GPRs (single PEBS record case), perf_sample_regs_user() modifies them to user space GPRs.
[ mingo: Clarified the changelog. ]
--- arch/x86/events/intel/ds.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
arch/x86/events/intel/ds.c +5 -3
@@ -1399,8 +1399,10 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)* + precise_ip < 2 for the non event IP * + For RTM TSX weight we need GPRs for the abort code. */ - gprs = (sample_type &PERF_SAMPLE_REGS_INTR ) && - (attr->sample_regs_intr &PEBS_GP_REGS ); + gprs = ((sample_type &PERF_SAMPLE_REGS_INTR ) && + (attr->sample_regs_intr &PEBS_GP_REGS )) || + ((sample_type &PERF_SAMPLE_REGS_USER ) && + (attr->sample_regs_user &PEBS_GP_REGS )); tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) && ((attr->config & INTEL_ARCH_EVENT_MASK) ==
@@ -2123,7 +2125,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, regs->flags &= ~PERF_EFLAGS_EXACT ; } -if (sample_type &PERF_SAMPLE_REGS_INTR ) +if (sample_type & (PERF_SAMPLE_REGS_INTR |PERF_SAMPLE_REGS_USER ))adaptive_pebs_save_regs (regs, gprs); }
Quoted 5 lines · Dapeng Mi
The below code would always unconditionally clear other status bits like perf metrics overflow bit once PEBS buffer overflows.
status &= intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
Quoted 7 lines
This is incorrect. Perf metrics overflow bit should be cleared only when fixed counter 3 in PEBS counter group. Otherwise perf metrics overflow could be missed to handle.
Fixes 7b2c05a15d29 ("perf/x86/intel: Generic support for hardware TopDown metrics")Signed-off-by Dapeng Mi <[email protected]>
For the two patches,
Thanks,
Kan
Quoted 51 lines
--- arch/x86/events/intel/core.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)arch/x86/events/intel/core.c +11
-2@@ -3140,7 +3140,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int bit;int handled =0 ; -u64 intel_ctrl =hybrid (cpuc->pmu, intel_ctrl);inc_irq_stat (apic_perf_irqs);@@ -3184,7 +3183,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data);static_call (x86_pmu_drain_pebs)(regs, &data); - status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/* * PMI throttle may be triggered, which stops the PEBS event. @@ -3195,6 +3193,15 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)*/ if (pebs_enabled != cpuc->pebs_enabled)wrmsrl (MSR_IA32_PEBS_ENABLE , cpuc->pebs_enabled); + +/* +* Above PEBS handler (PEBS counters snapshotting) has updated fixed +* counter 3 and perf metrics counts if they are in counter group, +* unnecessary to update again. +*/ +if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ] && +is_pebs_counter_event_group (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ])) + status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT ; }/* @@ -3214,6 +3221,8 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)static_call (intel_pmu_update_topdown_event)(NULL ,NULL ); } + status &=hybrid (cpuc->pmu, intel_ctrl); +/* * Checkpointed counters can lead to 'spurious' PMIs because the * rollback caused by the PMI will have cleared the overflow status base-commit:
5c3627b6f0595f1ec27e6f5df903bd072e9b9136
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: a5f5e1238f4ff919816f69e77d2537a48911767b Gitweb: https://git.kernel.org/tip/a5f5e1238f4ff919816f69e77d2537a48911767b Author: Dapeng Mi <[email protected]> AuthorDate: Tue, 15 Apr 2025 10:41:34 Committer: Ingo Molnar <[email protected]> CommitterDate: Thu, 17 Apr 2025 14:19:07 +02:00
perf/x86/intel: Don't clear perf metrics overflow bit unconditionally
The below code would always unconditionally clear other status bits like perf metrics overflow bit once PEBS buffer overflows:
status &= intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
This is incorrect. Perf metrics overflow bit should be cleared only when fixed counter 3 in PEBS counter group. Otherwise perf metrics overflow could be missed to handle.
--- arch/x86/events/intel/core.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
arch/x86/events/intel/core.c +11 -2
@@ -3049,7 +3049,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);int bit;int handled =0 ; -u64 intel_ctrl =hybrid (cpuc->pmu, intel_ctrl);inc_irq_stat (apic_perf_irqs);
@@ -3093,7 +3092,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data);static_call (x86_pmu_drain_pebs)(regs, &data); - status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/* * PMI throttle may be triggered, which stops the PEBS event.
@@ -3104,6 +3102,15 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)*/ if (pebs_enabled != cpuc->pebs_enabled)wrmsrl (MSR_IA32_PEBS_ENABLE , cpuc->pebs_enabled); + +/* +* Above PEBS handler (PEBS counters snapshotting) has updated fixed +* counter 3 and perf metrics counts if they are in counter group, +* unnecessary to update again. +*/ +if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ] && +is_pebs_counter_event_group (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS ])) + status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT ; }/*
@@ -3123,6 +3130,8 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)static_call (intel_pmu_update_topdown_event)(NULL ,NULL ); } + status &=hybrid (cpuc->pmu, intel_ctrl); +/* * Checkpointed counters can lead to 'spurious' PMIs because the * rollback caused by the PMI will have cleared the overflow status