perf/x86/intel: Prevent drain_pebs() reentry
Subsystems (2)
PATCH v2 1/1 perf/x86/intel: Prevent drain_pebs() reentry
The PEBS buffer is shared by all events on a CPU, so drain_pebs() must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.
Most invocations happen in NMI context, which naturally prevents reentry. However, drain_pebs() is also reachable from process context via intel_pmu_drain_pebs_buffer().
In those paths, the PMU is often already disabled, but not guaranteed. For example, __intel_pmu_pebs_disable() only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flight drain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()
__perf_event_stop()
x86_pmu_stop() (event->pmu->stop)
intel_pmu_disable_event()
intel_pmu_pebs_disable()
__intel_pmu_pebs_disable()
intel_pmu_drain_large_pebs()
intel_pmu_drain_pebs_buffer()Introduce __intel_pmu_quiesce() and __intel_pmu_resume() helpers and use them in intel_pmu_drain_large_pebs() to disable the full PMU around the intel_pmu_drain_pebs_buffer() call, preventing reentry.
Also add a warning in intel_pmu_drain_pebs_buffer() when the full PMU is not disabled.
---
v2: Move __intel_pmu_quiesce()/__intel_pmu_resume() pair into intel_pmu_drain_large_pebs() instead of guarding all drain_pebs() callbacks. (Peter)
v1: https://lore.kernel.org/all/[email protected]/
arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)
arch/x86/events/intel/core.c +24 -9
@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)this_cpu_ptr (&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce (void ) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int pmu_enabled = cpuc->enabled; + + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); + +return pmu_enabled; +} + +void __intel_pmu_resume (int pmu_enabled) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); +} +static int icl_set_topdown_event_period (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw;
@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN ) ||is_pebs_counter_event_group (event)) {struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); -bool pmu_enabled = cpuc->enabled; +int pmu_enabled;/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ ))return ; - cpuc->enabled =0 ; -if (pmu_enabled) -intel_pmu_disable_all (); - + pmu_enabled =__intel_pmu_quiesce ();/* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records.
@@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)static_call (intel_pmu_update_topdown_event)(event,NULL ); elseintel_pmu_drain_pebs_buffer (); - - cpuc->enabled = pmu_enabled; -if (pmu_enabled) -intel_pmu_enable_all (0 ); +__intel_pmu_resume (pmu_enabled);return ; }
arch/x86/events/intel/ds.c +7 -1
@@ -1242,8 +1242,11 @@ int intel_pmu_drain_bts_buffer(void)void intel_pmu_drain_pebs_buffer (void ) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data; +WARN_ON_ONCE (cpuc->enabled); +static_call (x86_pmu_drain_pebs)(NULL , &data); }
@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)static inline void intel_pmu_drain_large_pebs (struct cpu_hw_events *cpuc) {if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { +int enabled =__intel_pmu_quiesce ();intel_pmu_drain_pebs_buffer (); +__intel_pmu_resume (enabled); + } }static void __intel_pmu_pebs_enable (struct perf_event *event)
arch/x86/events/perf_event.h +3 -0
@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)wrmsrq (MSR_IA32_DEBUGCTLMSR , debugctl); } +extern int __intel_pmu_quiesce (void ); +extern void __intel_pmu_resume (int pmu_enabled); +int intel_pmu_save_and_restart (struct perf_event *event); struct event_constraint *
base-commit: 917d558b151cad5b05991e5eaee22efab33525ca
Signature · 1 line
2.34.1
[email protected]
AI reviewSashiko has reviewed this patch and found no issues. It looks great!
Quoted 13 lines
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
Quoted 8 lines
__perf_addr_filters_adjust()
perf_event_stop()
[ ... ]Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>
Sashiko has reviewed this patch and found no issues. It looks great!
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
Ian Rogers
reviewerQuoted 50 lines · Dapeng Mi
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()__perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer()Introduce
__intel_pmu_quiesce()and__intel_pmu_resume()helpers and use them inintel_pmu_drain_large_pebs()to disable the full PMU around theintel_pmu_drain_pebs_buffer()call, preventing reentry.Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>---
v2: Move __intel_pmu_quiesce()/__intel_pmu_resume() pair into
intel_pmu_drain_large_pebs()instead of guarding alldrain_pebs()callbacks. (Peter)v1: https://lore.kernel.org/all/[email protected]/
arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)arch/x86/events/intel/core.c +0
-0
arch/x86/events/intel/core.c static void intel_pmu_del_event(struct perf_event *event) · line 3128
Quoted 4 lines
arch/x86/events/intel/core.c +1
-0@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)this_cpu_ptr (&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce (void )
Thanks Dapeng. Rather than quiesce would __intel_pmu_disable_if_enabled be a more intention revealing name?
arch/x86/events/intel/core.c static void intel_pmu_del_event(struct perf_event *event) · line 3133
Quoted 5 lines
arch/x86/events/intel/core.c +5
-0+{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int pmu_enabled = cpuc->enabled; + + cpuc->enabled =0 ;
Presumably the line above could also be inside the if.
Quoted 19 lines
arch/x86/events/intel/core.c +15
-0+if (pmu_enabled) +intel_pmu_disable_all (); + +return pmu_enabled; +} + +void __intel_pmu_resume (int pmu_enabled) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); +} +static int icl_set_topdown_event_period (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw;
arch/x86/events/intel/core.c static void intel_pmu_read_event(struct perf_event *event) · line 3340
Quoted 5 lines
arch/x86/events/intel/core.c +1
-1@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN ) ||is_pebs_counter_event_group (event)) {struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); -bool pmu_enabled = cpuc->enabled; +int pmu_enabled;
nit: Maybe pmu_enabled would be better as pmu_was_enabled or pmu_reenable?
Quoted 42 lines
arch/x86/events/intel/core.c +2
-8/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ ))return ; - cpuc->enabled =0 ; -if (pmu_enabled) -intel_pmu_disable_all (); - + pmu_enabled =__intel_pmu_quiesce ();/* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)static_call (intel_pmu_update_topdown_event)(event,NULL ); elseintel_pmu_drain_pebs_buffer (); - - cpuc->enabled = pmu_enabled; -if (pmu_enabled) -intel_pmu_enable_all (0 ); +__intel_pmu_resume (pmu_enabled);return ; }arch/x86/events/intel/ds.c +3
-0@@ -1242,8 +1242,11 @@ int intel_pmu_drain_bts_buffer(void)void intel_pmu_drain_pebs_buffer (void ) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data; +WARN_ON_ONCE (cpuc->enabled); +static_call (x86_pmu_drain_pebs)(NULL , &data); }
arch/x86/events/intel/ds.c static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) · line 1871
Quoted 6 lines
arch/x86/events/intel/ds.c +2
-1@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)static inline void intel_pmu_drain_large_pebs (struct cpu_hw_events *cpuc) {if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { +int enabled =__intel_pmu_quiesce ();
Similarly, may be was_enabled rather than enabled.
Thanks,
Ian
Quoted 25 lines
arch/x86/events/intel/ds.c +2
-0intel_pmu_drain_pebs_buffer (); +__intel_pmu_resume (enabled); + } }static void __intel_pmu_pebs_enable (struct perf_event *event)arch/x86/events/perf_event.h +3
-0@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)wrmsrq (MSR_IA32_DEBUGCTLMSR , debugctl); } +extern int __intel_pmu_quiesce (void ); +extern void __intel_pmu_resume (int pmu_enabled); +int intel_pmu_save_and_restart (struct perf_event *event); struct event_constraint *base-commit:
917d558b151cad5b05991e5eaee22efab33525ca--
2.34.1
Quoted 53 lines · Ian Rogers
Quoted 52 lines · Dapeng Mi
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()__perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer()Introduce
__intel_pmu_quiesce()and__intel_pmu_resume()helpers and use them inintel_pmu_drain_large_pebs()to disable the full PMU around theintel_pmu_drain_pebs_buffer()call, preventing reentry.Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>---
v2: Move __intel_pmu_quiesce()/__intel_pmu_resume() pair into
intel_pmu_drain_large_pebs()instead of guarding alldrain_pebs()callbacks. (Peter)v1: https://lore.kernel.org/all/[email protected]/
arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)arch/x86/events/intel/core.c +0
-0@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)this_cpu_ptr (&cpu_hw_events)->n_late_setup--; }
Quoted 3 lines
arch/x86/events/intel/core.c static void intel_pmu_del_event(struct perf_event *event) · line 3128
Quoted 1 line
arch/x86/events/intel/core.c +1
-0+int __intel_pmu_quiesce (void )Thanks Dapeng. Rather than quiesce would
__intel_pmu_disable_if_enabledbe a more intention revealing name?
Thanks Ian for reviewing this patch.
Yeah, I was hesitated about the name for a while. Currently I still prefer the name __intel_pmu_quiesce() which is simplified enough than other names.
Quoted 7 lines
arch/x86/events/intel/core.c static void intel_pmu_del_event(struct perf_event *event) · line 3133
Quoted 6 lines
arch/x86/events/intel/core.c +5
-0+{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int pmu_enabled = cpuc->enabled; + + cpuc->enabled =0 ;Presumably the line above could also be inside the if.
Yes. I'm not sure if it's worthy to post a new version for this minor change. But If there is a new version, I would change it.
Quoted 18 lines
Quoted 18 lines
arch/x86/events/intel/core.c +15
-0+if (pmu_enabled) +intel_pmu_disable_all (); + +return pmu_enabled; +} + +void __intel_pmu_resume (int pmu_enabled) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); +} +static int icl_set_topdown_event_period (struct perf_event *event) {
Quoted 8 lines
arch/x86/events/intel/core.c static void intel_pmu_read_event(struct perf_event *event) · line 3340
Quoted 7 lines
arch/x86/events/intel/core.c +1
-1struct hw_perf_event *hwc = &event->hw;@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN ) ||is_pebs_counter_event_group (event)) {struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); -bool pmu_enabled = cpuc->enabled; +int pmu_enabled;nit: Maybe
pmu_enabledwould be better aspmu_was_enabledorpmu_reenable?
pmu_enabled seems better for me. pmu_enabled is simplified enough comparing with other names.
Thanks.
Quoted 78 lines
Quoted 42 lines
arch/x86/events/intel/core.c +2
-8/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ ))return ; - cpuc->enabled =0 ; -if (pmu_enabled) -intel_pmu_disable_all (); - + pmu_enabled =__intel_pmu_quiesce ();/* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)static_call (intel_pmu_update_topdown_event)(event,NULL ); elseintel_pmu_drain_pebs_buffer (); - - cpuc->enabled = pmu_enabled; -if (pmu_enabled) -intel_pmu_enable_all (0 ); +__intel_pmu_resume (pmu_enabled);return ; }arch/x86/events/intel/ds.c +3
-0@@ -1242,8 +1242,11 @@ int intel_pmu_drain_bts_buffer(void)void intel_pmu_drain_pebs_buffer (void ) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data; +WARN_ON_ONCE (cpuc->enabled); +static_call (x86_pmu_drain_pebs)(NULL , &data); }arch/x86/events/intel/ds.c static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) · line 1871
Quoted 6 lines
arch/x86/events/intel/ds.c +2
-1@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)static inline void intel_pmu_drain_large_pebs (struct cpu_hw_events *cpuc) {if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { +int enabled =__intel_pmu_quiesce ();Similarly, may be
was_enabledrather than enabled.Thanks,
IanQuoted 25 lines
arch/x86/events/intel/ds.c +2
-0intel_pmu_drain_pebs_buffer (); +__intel_pmu_resume (enabled); + } }static void __intel_pmu_pebs_enable (struct perf_event *event)arch/x86/events/perf_event.h +3
-0@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)wrmsrq (MSR_IA32_DEBUGCTLMSR , debugctl); } +extern int __intel_pmu_quiesce (void ); +extern void __intel_pmu_resume (int pmu_enabled); +int intel_pmu_save_and_restart (struct perf_event *event); struct event_constraint *base-commit:
917d558b151cad5b05991e5eaee22efab33525ca--
2.34.1
Hi Peter,
Could you please review and queue this patch if it's good enough? This version addresses your comments. Thanks.
Quoted 152 lines · Dapeng Mi
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()__perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer()Introduce
__intel_pmu_quiesce()and__intel_pmu_resume()helpers and use them inintel_pmu_drain_large_pebs()to disable the full PMU around theintel_pmu_drain_pebs_buffer()call, preventing reentry.Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>---
v2: Move __intel_pmu_quiesce()/__intel_pmu_resume() pair into
intel_pmu_drain_large_pebs()instead of guarding alldrain_pebs()callbacks. (Peter)v1: https://lore.kernel.org/all/[email protected]/
arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)arch/x86/events/intel/core.c +24
-9@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)this_cpu_ptr (&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce (void ) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int pmu_enabled = cpuc->enabled; + + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); + +return pmu_enabled; +} + +void __intel_pmu_resume (int pmu_enabled) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); +} +static int icl_set_topdown_event_period (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw;@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN ) ||is_pebs_counter_event_group (event)) {struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); -bool pmu_enabled = cpuc->enabled; +int pmu_enabled;/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ ))return ; - cpuc->enabled =0 ; -if (pmu_enabled) -intel_pmu_disable_all (); - + pmu_enabled =__intel_pmu_quiesce ();/* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)static_call (intel_pmu_update_topdown_event)(event,NULL ); elseintel_pmu_drain_pebs_buffer (); - - cpuc->enabled = pmu_enabled; -if (pmu_enabled) -intel_pmu_enable_all (0 ); +__intel_pmu_resume (pmu_enabled);return ; }arch/x86/events/intel/ds.c +7
-1@@ -1242,8 +1242,11 @@ int intel_pmu_drain_bts_buffer(void)void intel_pmu_drain_pebs_buffer (void ) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data; +WARN_ON_ONCE (cpuc->enabled); +static_call (x86_pmu_drain_pebs)(NULL , &data); }@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)static inline void intel_pmu_drain_large_pebs (struct cpu_hw_events *cpuc) {if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { +int enabled =__intel_pmu_quiesce ();intel_pmu_drain_pebs_buffer (); +__intel_pmu_resume (enabled); + } }static void __intel_pmu_pebs_enable (struct perf_event *event)arch/x86/events/perf_event.h +3
-0@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)wrmsrq (MSR_IA32_DEBUGCTLMSR , debugctl); } +extern int __intel_pmu_quiesce (void ); +extern void __intel_pmu_resume (int pmu_enabled); +int intel_pmu_save_and_restart (struct perf_event *event); struct event_constraint *base-commit:
917d558b151cad5b05991e5eaee22efab33525ca
Peter Zijlstra
maintainerQuoted 38 lines · Dapeng
Hi Peter,
Could you please review and queue this patch if it's good enough? This version addresses your comments. Thanks.
Quoted 31 lines · Dapeng Mi
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()__perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer()Introduce
__intel_pmu_quiesce()and__intel_pmu_resume()helpers and use them inintel_pmu_drain_large_pebs()to disable the full PMU around theintel_pmu_drain_pebs_buffer()call, preventing reentry.Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>
Ah yes. Can you get me a Fixes tag to go with this?
Quoted 35 lines · Peter Zijlstra
Quoted 34 lines · Dapeng
Hi Peter,
Could you please review and queue this patch if it's good enough? This version addresses your comments. Thanks.
Quoted 27 lines · Dapeng Mi
The PEBS buffer is shared by all events on a CPU, so
drain_pebs()must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.Most invocations happen in NMI context, which naturally prevents reentry. However,
drain_pebs()is also reachable from process context viaintel_pmu_drain_pebs_buffer().In those paths, the PMU is often already disabled, but not guaranteed. For example,
__intel_pmu_pebs_disable()only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flightdrain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()__perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer()Introduce
__intel_pmu_quiesce()and__intel_pmu_resume()helpers and use them inintel_pmu_drain_large_pebs()to disable the full PMU around theintel_pmu_drain_pebs_buffer()call, preventing reentry.
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
Also add a warning in
intel_pmu_drain_pebs_buffer()when the full PMU is not disabled.Signed-off-by Dapeng Mi <[email protected]>Ah yes. Can you get me a Fixes tag to go with this?
Sure. Here it is. Thanks.
PEBS_DATA_CFG")
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: a56c03a397e2cd0c4cf8da96dcd6214f7d0e7d8c Gitweb: https://git.kernel.org/tip/a56c03a397e2cd0c4cf8da96dcd6214f7d0e7d8c Author: Dapeng Mi <[email protected]> AuthorDate: Thu, 13 Aug 2026 14:43:46 +08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Thu, 10 Sep 2026 10:22:50 +02:00
perf/x86/intel: Prevent drain_pebs() reentry
The PEBS buffer is shared by all events on a CPU, so drain_pebs() must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory.
Most invocations happen in NMI context, which naturally prevents reentry. However, drain_pebs() is also reachable from process context via intel_pmu_drain_pebs_buffer().
In those paths, the PMU is often already disabled, but not guaranteed. For example, __intel_pmu_pebs_disable() only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flight drain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()
__perf_event_stop()
x86_pmu_stop() (event->pmu->stop)
intel_pmu_disable_event()
intel_pmu_pebs_disable()
__intel_pmu_pebs_disable()
intel_pmu_drain_large_pebs()
intel_pmu_drain_pebs_buffer()Introduce __intel_pmu_quiesce() and __intel_pmu_resume() helpers and use them in intel_pmu_drain_large_pebs() to disable the full PMU around the intel_pmu_drain_pebs_buffer() call, preventing reentry.
Also add a warning in intel_pmu_drain_pebs_buffer() when the full PMU is not disabled.
--- arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)
arch/x86/events/intel/core.c +24 -9
@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)this_cpu_ptr (&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce (void ) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int pmu_enabled = cpuc->enabled; + + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); + +return pmu_enabled; +} + +void __intel_pmu_resume (int pmu_enabled) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); +} +static int icl_set_topdown_event_period (struct perf_event *event) {struct hw_perf_event *hwc = &event->hw;
@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN ) ||is_pebs_counter_event_group (event)) {struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); -bool pmu_enabled = cpuc->enabled; +int pmu_enabled;/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ ))return ; - cpuc->enabled =0 ; -if (pmu_enabled) -intel_pmu_disable_all (); - + pmu_enabled =__intel_pmu_quiesce ();/* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records.
@@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)static_call (intel_pmu_update_topdown_event)(event,NULL ); elseintel_pmu_drain_pebs_buffer (); - - cpuc->enabled = pmu_enabled; -if (pmu_enabled) -intel_pmu_enable_all (0 ); +__intel_pmu_resume (pmu_enabled);return ; }
arch/x86/events/intel/ds.c +7 -1
@@ -1242,8 +1242,11 @@ unlock:void intel_pmu_drain_pebs_buffer (void ) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct perf_sample_data data; +WARN_ON_ONCE (cpuc->enabled); +static_call (x86_pmu_drain_pebs)(NULL , &data); }
@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)static inline void intel_pmu_drain_large_pebs (struct cpu_hw_events *cpuc) {if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { +int enabled =__intel_pmu_quiesce ();intel_pmu_drain_pebs_buffer (); +__intel_pmu_resume (enabled); + } }static void __intel_pmu_pebs_enable (struct perf_event *event)
arch/x86/events/perf_event.h +3 -0
@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)wrmsrq (MSR_IA32_DEBUGCTLMSR , debugctl); } +extern int __intel_pmu_quiesce (void ); +extern void __intel_pmu_resume (int pmu_enabled); +int intel_pmu_save_and_restart (struct perf_event *event); struct event_constraint *