perf/x86/intel: Apply static call for drain_pebs
Subsystems (3)
PATCH v10 1/4 perf/x86/intel: Apply static call for drain_pebs
From: "Peter Zijlstra (Intel)" <[email protected]>
The x86_pmu_drain_pebs static call was introduced in commit 7c9903c9bf71 ("x86/perf, static_call: Optimize x86_pmu methods"), but it's not really used to replace the old method.
Apply the static call for drain_pebs.
---
New for V10
arch/x86/events/intel/core.c | 2 +- arch/x86/events/intel/ds.c | 2 +- arch/x86/events/perf_event.h | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -3066,7 +3066,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data); - x86_pmu.drain_pebs (regs, &data); +static_call (x86_pmu_drain_pebs)(regs, &data); status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/*
arch/x86/events/intel/ds.c +1 -1
@@ -957,7 +957,7 @@ static inline void intel_pmu_drain_pebs_buffer(void) {struct perf_sample_data data; - x86_pmu.drain_pebs (NULL , &data); +static_call (x86_pmu_drain_pebs)(NULL , &data); }/*
arch/x86/events/perf_event.h +1 -0
@@ -1107,6 +1107,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update); +DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {
Signature · 1 line
2.38.1
PATCH v10 2/4 perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read
From: Kan Liang <[email protected]>
The WARN_ON(this_cpu_read(cpu_hw_events.enabled)) in the intel_pmu_save_and_restart_reload() is triggered, when sampling read topdown events.
In a NMI handler, the cpu_hw_events.enabled is set and used to indicate the status of core PMU. The generic pmu->pmu_disable_count, updated in the perf_pmu_disable/enable pair, is not touched.
However, the perf_pmu_disable/enable pair is invoked when sampling read in a NMI handler. The cpuc->enabled is mistakenly set by the perf_pmu_enable().
Avoid disabling PMU if the core PMU is already disabled. Merge the logic together.
---
Changes since V9:
- Merge the read_event related codes together
arch/x86/events/intel/core.c | 41 ++++++++++++++++++++---------------- arch/x86/events/intel/ds.c | 11 +--------- arch/x86/events/perf_event.h | 2 +- 3 files changed, 25 insertions(+), 29 deletions(-)
arch/x86/events/intel/core.c +23 -18
@@ -2775,28 +2775,33 @@ static u64 icl_update_topdown_event(struct perf_event *event)DEFINE_STATIC_CALL (intel_pmu_update_topdown_event, x86_perf_event_update); -static void intel_pmu_read_topdown_event (struct perf_event *event) +static void intel_pmu_read_event (struct perf_event *event) { -struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN )) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +bool pmu_enabled = cpuc->enabled; -/* Only need to call update_topdown_event() once for group read. */ -if ((cpuc->txn_flags &PERF_PMU_TXN_READ ) && - !is_slots_event (event)) -return ; +/* Only need to call update_topdown_event() once for group read. */ +if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ )) +return ; -perf_pmu_disable (event->pmu); -static_call (intel_pmu_update_topdown_event)(event); -perf_pmu_enable (event->pmu); -} + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); -static void intel_pmu_read_event (struct perf_event *event) -{ -if (event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD ) -intel_pmu_auto_reload_read (event); -else if (is_topdown_count (event)) -intel_pmu_read_topdown_event (event); -else -x86_perf_event_update (event); +if (is_topdown_event (event)) +static_call (intel_pmu_update_topdown_event)(event); +else +intel_pmu_drain_pebs_buffer (); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); + +return ; + } + +x86_perf_event_update (event); }static void intel_pmu_enable_fixed (struct perf_event *event)
arch/x86/events/intel/ds.c +1 -10
@@ -953,7 +953,7 @@ int intel_pmu_drain_bts_buffer(void)return 1 ; } -static inline void intel_pmu_drain_pebs_buffer (void ) +void intel_pmu_drain_pebs_buffer (void ) {struct perf_sample_data data;
@@ -2094,15 +2094,6 @@ get_next_pebs_record_by_bit(void *base, void *top, int bit)return NULL ; } -void intel_pmu_auto_reload_read (struct perf_event *event) -{ -WARN_ON (!(event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD )); - -perf_pmu_disable (event->pmu); -intel_pmu_drain_pebs_buffer (); -perf_pmu_enable (event->pmu); -} -/* * Special variant of intel_pmu_save_and_restart() for auto-reload. */
arch/x86/events/perf_event.h +1 -1
@@ -1644,7 +1644,7 @@ void intel_pmu_pebs_disable_all(void);void intel_pmu_pebs_sched_task (struct perf_event_pmu_context *pmu_ctx,bool sched_in); -void intel_pmu_auto_reload_read (struct perf_event *event); +void intel_pmu_drain_pebs_buffer (void );void intel_pmu_store_pebs_lbrs (struct lbr_entry *lbr);
Signature · 1 line
2.38.1
[tip: perf/core] perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 5c32ae8ac8936b992fc0069c6438df932a753325 Gitweb: https://git.kernel.org/tip/5c32ae8ac8936b992fc0069c6438df932a753325 Author: Kan Liang <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:01 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 03 Feb 2025 11:46:07 +01:00
perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read
The WARN_ON(this_cpu_read(cpu_hw_events.enabled)) in the intel_pmu_save_and_restart_reload() is triggered, when sampling read topdown events.
In a NMI handler, the cpu_hw_events.enabled is set and used to indicate the status of core PMU. The generic pmu->pmu_disable_count, updated in the perf_pmu_disable/enable pair, is not touched.
However, the perf_pmu_disable/enable pair is invoked when sampling read in a NMI handler. The cpuc->enabled is mistakenly set by the perf_pmu_enable().
Avoid disabling PMU if the core PMU is already disabled. Merge the logic together.
--- arch/x86/events/intel/core.c | 41 +++++++++++++++++++---------------- arch/x86/events/intel/ds.c | 11 +--------- arch/x86/events/perf_event.h | 2 +- 3 files changed, 25 insertions(+), 29 deletions(-)
arch/x86/events/intel/core.c +23 -18
@@ -2785,28 +2785,33 @@ static u64 icl_update_topdown_event(struct perf_event *event)DEFINE_STATIC_CALL (intel_pmu_update_topdown_event, x86_perf_event_update); -static void intel_pmu_read_topdown_event (struct perf_event *event) +static void intel_pmu_read_event (struct perf_event *event) { -struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN )) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +bool pmu_enabled = cpuc->enabled; -/* Only need to call update_topdown_event() once for group read. */ -if ((cpuc->txn_flags &PERF_PMU_TXN_READ ) && - !is_slots_event (event)) -return ; +/* Only need to call update_topdown_event() once for group read. */ +if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ )) +return ; -perf_pmu_disable (event->pmu); -static_call (intel_pmu_update_topdown_event)(event); -perf_pmu_enable (event->pmu); -} + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); -static void intel_pmu_read_event (struct perf_event *event) -{ -if (event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD ) -intel_pmu_auto_reload_read (event); -else if (is_topdown_count (event)) -intel_pmu_read_topdown_event (event); -else -x86_perf_event_update (event); +if (is_topdown_event (event)) +static_call (intel_pmu_update_topdown_event)(event); +else +intel_pmu_drain_pebs_buffer (); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); + +return ; + } + +x86_perf_event_update (event); }static void intel_pmu_enable_fixed (struct perf_event *event)
arch/x86/events/intel/ds.c +1 -10
@@ -953,7 +953,7 @@ unlock:return 1 ; } -static inline void intel_pmu_drain_pebs_buffer (void ) +void intel_pmu_drain_pebs_buffer (void ) {struct perf_sample_data data;
@@ -2094,15 +2094,6 @@ get_next_pebs_record_by_bit(void *base, void *top, int bit)return NULL ; } -void intel_pmu_auto_reload_read (struct perf_event *event) -{ -WARN_ON (!(event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD )); - -perf_pmu_disable (event->pmu); -intel_pmu_drain_pebs_buffer (); -perf_pmu_enable (event->pmu); -} -/* * Special variant of intel_pmu_save_and_restart() for auto-reload. */
arch/x86/events/perf_event.h +1 -1
@@ -1644,7 +1644,7 @@ void intel_pmu_pebs_disable_all(void);void intel_pmu_pebs_sched_task (struct perf_event_pmu_context *pmu_ctx,bool sched_in); -void intel_pmu_auto_reload_read (struct perf_event *event); +void intel_pmu_drain_pebs_buffer (void );void intel_pmu_store_pebs_lbrs (struct lbr_entry *lbr);
[tip: perf/core] perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read
In reply to [email protected]The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: f9bdf1f953392c9edd69a7f884f78c0390127029 Gitweb: https://git.kernel.org/tip/f9bdf1f953392c9edd69a7f884f78c0390127029 Author: Kan Liang <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:01 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 05 Feb 2025 10:29:45 +01:00
perf/x86/intel: Avoid disable PMU if !cpuc->enabled in sample read
The WARN_ON(this_cpu_read(cpu_hw_events.enabled)) in the intel_pmu_save_and_restart_reload() is triggered, when sampling read topdown events.
In a NMI handler, the cpu_hw_events.enabled is set and used to indicate the status of core PMU. The generic pmu->pmu_disable_count, updated in the perf_pmu_disable/enable pair, is not touched.
However, the perf_pmu_disable/enable pair is invoked when sampling read in a NMI handler. The cpuc->enabled is mistakenly set by the perf_pmu_enable().
Avoid disabling PMU if the core PMU is already disabled. Merge the logic together.
--- arch/x86/events/intel/core.c | 41 +++++++++++++++++++---------------- arch/x86/events/intel/ds.c | 11 +--------- arch/x86/events/perf_event.h | 2 +- 3 files changed, 25 insertions(+), 29 deletions(-)
arch/x86/events/intel/core.c +23 -18
@@ -2785,28 +2785,33 @@ static u64 icl_update_topdown_event(struct perf_event *event)DEFINE_STATIC_CALL (intel_pmu_update_topdown_event, x86_perf_event_update); -static void intel_pmu_read_topdown_event (struct perf_event *event) +static void intel_pmu_read_event (struct perf_event *event) { -struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD |PERF_X86_EVENT_TOPDOWN )) { +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +bool pmu_enabled = cpuc->enabled; -/* Only need to call update_topdown_event() once for group read. */ -if ((cpuc->txn_flags &PERF_PMU_TXN_READ ) && - !is_slots_event (event)) -return ; +/* Only need to call update_topdown_event() once for group read. */ +if (is_metric_event (event) && (cpuc->txn_flags &PERF_PMU_TXN_READ )) +return ; -perf_pmu_disable (event->pmu); -static_call (intel_pmu_update_topdown_event)(event); -perf_pmu_enable (event->pmu); -} + cpuc->enabled =0 ; +if (pmu_enabled) +intel_pmu_disable_all (); -static void intel_pmu_read_event (struct perf_event *event) -{ -if (event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD ) -intel_pmu_auto_reload_read (event); -else if (is_topdown_count (event)) -intel_pmu_read_topdown_event (event); -else -x86_perf_event_update (event); +if (is_topdown_event (event)) +static_call (intel_pmu_update_topdown_event)(event); +else +intel_pmu_drain_pebs_buffer (); + + cpuc->enabled = pmu_enabled; +if (pmu_enabled) +intel_pmu_enable_all (0 ); + +return ; + } + +x86_perf_event_update (event); }static void intel_pmu_enable_fixed (struct perf_event *event)
arch/x86/events/intel/ds.c +1 -10
@@ -953,7 +953,7 @@ unlock:return 1 ; } -static inline void intel_pmu_drain_pebs_buffer (void ) +void intel_pmu_drain_pebs_buffer (void ) {struct perf_sample_data data;
@@ -2094,15 +2094,6 @@ get_next_pebs_record_by_bit(void *base, void *top, int bit)return NULL ; } -void intel_pmu_auto_reload_read (struct perf_event *event) -{ -WARN_ON (!(event->hw.flags &PERF_X86_EVENT_AUTO_RELOAD )); - -perf_pmu_disable (event->pmu); -intel_pmu_drain_pebs_buffer (); -perf_pmu_enable (event->pmu); -} -/* * Special variant of intel_pmu_save_and_restart() for auto-reload. */
arch/x86/events/perf_event.h +1 -1
@@ -1644,7 +1644,7 @@ void intel_pmu_pebs_disable_all(void);void intel_pmu_pebs_sched_task (struct perf_event_pmu_context *pmu_ctx,bool sched_in); -void intel_pmu_auto_reload_read (struct perf_event *event); +void intel_pmu_drain_pebs_buffer (void );void intel_pmu_store_pebs_lbrs (struct lbr_entry *lbr);
PATCH v10 3/4 perf: Avoid the read if the count is already updated
In reply to [email protected]From: "Peter Zijlstra (Intel)" <[email protected]>
The event may have been updated in the PMU-specific implementation, e.g., Intel PEBS counters snapshotting. The common code should not read and overwrite the value.
The PERF_SAMPLE_READ in the data->sample_type can be used to detect whether the PMU-specific value is available. If yes, avoid the pmu->read() in the common code. Add a new flag, skip_read, to track the case.
Factor out a perf_pmu_read() to clean up the code.
---
No changes since V9
include/linux/perf_event.h | 8 +++++++- kernel/events/core.c | 33 ++++++++++++++++----------------- kernel/events/ring_buffer.c | 1 + 3 files changed, 24 insertions(+), 18 deletions(-)
include/linux/perf_event.h +7 -1
@@ -1062,7 +1062,13 @@ struct perf_output_handle {struct perf_buffer *rb;unsigned long wakeup;unsigned long size; -u64 aux_flags; +union { +u64 flags;/* perf_output*() */ +u64 aux_flags;/* perf_aux_output*() */ +struct { +u64 skip_read :1 ; + }; + };union {void *addr;unsigned long head;
kernel/events/core.c +16 -17
@@ -1191,6 +1191,12 @@ static void perf_assert_pmu_disabled(struct pmu *pmu)WARN_ON_ONCE (*this_cpu_ptr (pmu->pmu_disable_count) ==0 ); } +static inline void perf_pmu_read (struct perf_event *event) +{ +if (event->state ==PERF_EVENT_STATE_ACTIVE ) + event->pmu->read (event); +} +static void get_ctx (struct perf_event_context *ctx) {refcount_inc (&ctx->refcount);
@@ -3473,8 +3479,7 @@ static void __perf_event_sync_stat(struct perf_event *event,* we know the event must be on the current CPU, therefore we * don't need to use it. */ -if (event->state ==PERF_EVENT_STATE_ACTIVE ) - event->pmu->read (event); +perf_pmu_read (event);perf_event_update_time (event);
@@ -4618,15 +4623,8 @@ static void __perf_event_read(void *info) pmu->read (event); -for_each_sibling_event (sub, event) { -if (sub->state ==PERF_EVENT_STATE_ACTIVE ) { -/* -* Use sibling's PMU rather than @event's since -* sibling could be on different (eg: software) PMU. -*/ - sub->pmu->read (sub); - } - } +for_each_sibling_event (sub, event) +perf_pmu_read (sub); data->ret = pmu->commit_txn (pmu);
@@ -7400,9 +7398,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,if (read_format &PERF_FORMAT_TOTAL_TIME_RUNNING ) values[n++] = running; -if ((leader != event) && - (leader->state ==PERF_EVENT_STATE_ACTIVE )) - leader->pmu->read (leader); +if ((leader != event) && !handle->skip_read) +perf_pmu_read (leader); values[n++] =perf_event_count (leader, self);if (read_format &PERF_FORMAT_ID )
@@ -7415,9 +7412,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,for_each_sibling_event (sub, leader) { n =0 ; -if ((sub != event) && - (sub->state ==PERF_EVENT_STATE_ACTIVE )) - sub->pmu->read (sub); +if ((sub != event) && !handle->skip_read) +perf_pmu_read (sub); values[n++] =perf_event_count (sub, self);if (read_format &PERF_FORMAT_ID )
@@ -7476,6 +7472,9 @@ void perf_output_sample(struct perf_output_handle *handle, {u64 sample_type = data->type; +if (data->sample_flags &PERF_SAMPLE_READ ) + handle->skip_read =1 ; +perf_output_put (handle, *header);if (sample_type &PERF_SAMPLE_IDENTIFIER )
kernel/events/ring_buffer.c +1 -0
@@ -185,6 +185,7 @@ __perf_output_begin(struct perf_output_handle *handle, handle->rb = rb; handle->event = event; + handle->flags =0 ; have_lost =local_read (&rb->lost);if (unlikely (have_lost)) {
Signature · 1 line
2.38.1
[tip: perf/core] perf: Avoid the read if the count is already updated
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 186dbd9cd489e7f895f909d336c58e4ba2f660a1 Gitweb: https://git.kernel.org/tip/186dbd9cd489e7f895f909d336c58e4ba2f660a1 Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:02 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 03 Feb 2025 11:46:07 +01:00
perf: Avoid the read if the count is already updated
The event may have been updated in the PMU-specific implementation, e.g., Intel PEBS counters snapshotting. The common code should not read and overwrite the value.
The PERF_SAMPLE_READ in the data->sample_type can be used to detect whether the PMU-specific value is available. If yes, avoid the pmu->read() in the common code. Add a new flag, skip_read, to track the case.
Factor out a perf_pmu_read() to clean up the code.
--- include/linux/perf_event.h | 8 +++++++- kernel/events/core.c | 33 ++++++++++++++++----------------- kernel/events/ring_buffer.c | 1 + 3 files changed, 24 insertions(+), 18 deletions(-)
include/linux/perf_event.h +7 -1
@@ -1062,7 +1062,13 @@ struct perf_output_handle {struct perf_buffer *rb;unsigned long wakeup;unsigned long size; -u64 aux_flags; +union { +u64 flags;/* perf_output*() */ +u64 aux_flags;/* perf_aux_output*() */ +struct { +u64 skip_read :1 ; + }; + };union {void *addr;unsigned long head;
kernel/events/core.c +16 -17
@@ -1191,6 +1191,12 @@ static void perf_assert_pmu_disabled(struct pmu *pmu)WARN_ON_ONCE (*this_cpu_ptr (pmu->pmu_disable_count) ==0 ); } +static inline void perf_pmu_read (struct perf_event *event) +{ +if (event->state ==PERF_EVENT_STATE_ACTIVE ) + event->pmu->read (event); +} +static void get_ctx (struct perf_event_context *ctx) {refcount_inc (&ctx->refcount);
@@ -3473,8 +3479,7 @@ static void __perf_event_sync_stat(struct perf_event *event,* we know the event must be on the current CPU, therefore we * don't need to use it. */ -if (event->state ==PERF_EVENT_STATE_ACTIVE ) - event->pmu->read (event); +perf_pmu_read (event);perf_event_update_time (event);
@@ -4618,15 +4623,8 @@ static void __perf_event_read(void *info) pmu->read (event); -for_each_sibling_event (sub, event) { -if (sub->state ==PERF_EVENT_STATE_ACTIVE ) { -/* -* Use sibling's PMU rather than @event's since -* sibling could be on different (eg: software) PMU. -*/ - sub->pmu->read (sub); - } - } +for_each_sibling_event (sub, event) +perf_pmu_read (sub); data->ret = pmu->commit_txn (pmu);
@@ -7444,9 +7442,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,if (read_format &PERF_FORMAT_TOTAL_TIME_RUNNING ) values[n++] = running; -if ((leader != event) && - (leader->state ==PERF_EVENT_STATE_ACTIVE )) - leader->pmu->read (leader); +if ((leader != event) && !handle->skip_read) +perf_pmu_read (leader); values[n++] =perf_event_count (leader, self);if (read_format &PERF_FORMAT_ID )
@@ -7459,9 +7456,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,for_each_sibling_event (sub, leader) { n =0 ; -if ((sub != event) && - (sub->state ==PERF_EVENT_STATE_ACTIVE )) - sub->pmu->read (sub); +if ((sub != event) && !handle->skip_read) +perf_pmu_read (sub); values[n++] =perf_event_count (sub, self);if (read_format &PERF_FORMAT_ID )
@@ -7520,6 +7516,9 @@ void perf_output_sample(struct perf_output_handle *handle, {u64 sample_type = data->type; +if (data->sample_flags &PERF_SAMPLE_READ ) + handle->skip_read =1 ; +perf_output_put (handle, *header);if (sample_type &PERF_SAMPLE_IDENTIFIER )
kernel/events/ring_buffer.c +1 -0
@@ -185,6 +185,7 @@ __perf_output_begin(struct perf_output_handle *handle, handle->rb = rb; handle->event = event; + handle->flags =0 ; have_lost =local_read (&rb->lost);if (unlikely (have_lost)) {
[tip: perf/core] perf: Avoid the read if the count is already updated
In reply to [email protected]The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 8ce939a0fa194939cc1f92dbd8bc1a7806e7d40a Gitweb: https://git.kernel.org/tip/8ce939a0fa194939cc1f92dbd8bc1a7806e7d40a Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:02 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 05 Feb 2025 10:29:45 +01:00
perf: Avoid the read if the count is already updated
The event may have been updated in the PMU-specific implementation, e.g., Intel PEBS counters snapshotting. The common code should not read and overwrite the value.
The PERF_SAMPLE_READ in the data->sample_type can be used to detect whether the PMU-specific value is available. If yes, avoid the pmu->read() in the common code. Add a new flag, skip_read, to track the case.
Factor out a perf_pmu_read() to clean up the code.
--- include/linux/perf_event.h | 8 +++++++- kernel/events/core.c | 33 ++++++++++++++++----------------- kernel/events/ring_buffer.c | 1 + 3 files changed, 24 insertions(+), 18 deletions(-)
include/linux/perf_event.h +7 -1
@@ -1062,7 +1062,13 @@ struct perf_output_handle {struct perf_buffer *rb;unsigned long wakeup;unsigned long size; -u64 aux_flags; +union { +u64 flags;/* perf_output*() */ +u64 aux_flags;/* perf_aux_output*() */ +struct { +u64 skip_read :1 ; + }; + };union {void *addr;unsigned long head;
kernel/events/core.c +16 -17
@@ -1191,6 +1191,12 @@ static void perf_assert_pmu_disabled(struct pmu *pmu)WARN_ON_ONCE (*this_cpu_ptr (pmu->pmu_disable_count) ==0 ); } +static inline void perf_pmu_read (struct perf_event *event) +{ +if (event->state ==PERF_EVENT_STATE_ACTIVE ) + event->pmu->read (event); +} +static void get_ctx (struct perf_event_context *ctx) {refcount_inc (&ctx->refcount);
@@ -3473,8 +3479,7 @@ static void __perf_event_sync_stat(struct perf_event *event,* we know the event must be on the current CPU, therefore we * don't need to use it. */ -if (event->state ==PERF_EVENT_STATE_ACTIVE ) - event->pmu->read (event); +perf_pmu_read (event);perf_event_update_time (event);
@@ -4618,15 +4623,8 @@ static void __perf_event_read(void *info) pmu->read (event); -for_each_sibling_event (sub, event) { -if (sub->state ==PERF_EVENT_STATE_ACTIVE ) { -/* -* Use sibling's PMU rather than @event's since -* sibling could be on different (eg: software) PMU. -*/ - sub->pmu->read (sub); - } - } +for_each_sibling_event (sub, event) +perf_pmu_read (sub); data->ret = pmu->commit_txn (pmu);
@@ -7444,9 +7442,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,if (read_format &PERF_FORMAT_TOTAL_TIME_RUNNING ) values[n++] = running; -if ((leader != event) && - (leader->state ==PERF_EVENT_STATE_ACTIVE )) - leader->pmu->read (leader); +if ((leader != event) && !handle->skip_read) +perf_pmu_read (leader); values[n++] =perf_event_count (leader, self);if (read_format &PERF_FORMAT_ID )
@@ -7459,9 +7456,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,for_each_sibling_event (sub, leader) { n =0 ; -if ((sub != event) && - (sub->state ==PERF_EVENT_STATE_ACTIVE )) - sub->pmu->read (sub); +if ((sub != event) && !handle->skip_read) +perf_pmu_read (sub); values[n++] =perf_event_count (sub, self);if (read_format &PERF_FORMAT_ID )
@@ -7520,6 +7516,9 @@ void perf_output_sample(struct perf_output_handle *handle, {u64 sample_type = data->type; +if (data->sample_flags &PERF_SAMPLE_READ ) + handle->skip_read =1 ; +perf_output_put (handle, *header);if (sample_type &PERF_SAMPLE_IDENTIFIER )
kernel/events/ring_buffer.c +1 -0
@@ -185,6 +185,7 @@ __perf_output_begin(struct perf_output_handle *handle, handle->rb = rb; handle->event = event; + handle->flags =0 ; have_lost =local_read (&rb->lost);if (unlikely (have_lost)) {
PATCH v10 4/4 perf/x86/intel: Support PEBS counters snapshotting
In reply to [email protected]From: Kan Liang <[email protected]>
The counters snapshotting is a new adaptive PEBS extension, which can capture programmable counters, fixed-function counters, and performance metrics in a PEBS record. The feature is available in the PEBS format V6.
The target counters can be configured in the new fields of MSR_PEBS_CFG. Then the PEBS HW will generate the bit mask of counters (Counters Group Header) followed by the content of all the requested counters into a PEBS record.
The current Linux perf sample read feature can read all events in the group when any event in the group is overflowed. But the rdpmc in the NMI/overflow handler has a small gap from overflow. Also, there is some overhead for each rdpmc read. The counters snapshotting feature can be used as an accurate and low-overhead replacement.
Extend intel_update_topdown_event() to accept the value from PEBS records.
Add a new PEBS_CNTR flag to indicate a sample read group that utilizes the counters snapshotting feature. When the group is scheduled, the PEBS configure can be updated accordingly.
To prevent the case that a PEBS record value might be in the past relative to what is already in the event, perf always stops the PMU and drains the PEBS buffer before updating the corresponding event->count.
---
Changes since V9:
- Remove the stale Reviewed-by tag
- Add late_setup static call in struct x86_pmu
- Rename and move the late_setup to the begin of x86_pmu_enable()
- Update the comment in intel_perf_event_update_pmc() to explain the case of HES_STOPPED counter.
- Move INTEL_PMC_IDX_FIXED out of PEBS_DATACFG_FIX_BIT
- Rename intel_perf_event_pmc_to_count() intel_perf_event_update_pmc()
- Add a helper function __setup_pebs_counter_group() to avoid goto
arch/x86/events/core.c | 13 ++ arch/x86/events/intel/core.c | 75 ++++++++--- arch/x86/events/intel/ds.c | 191 +++++++++++++++++++++++++++-- arch/x86/events/perf_event.h | 13 ++ arch/x86/events/perf_event_flags.h | 2 +- arch/x86/include/asm/perf_event.h | 15 +++ 6 files changed, 284 insertions(+), 25 deletions(-)
arch/x86/events/core.c +13 -0
@@ -94,6 +94,8 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);DEFINE_STATIC_CALL_NULL (x86_pmu_filter, *x86_pmu.filter); +DEFINE_STATIC_CALL_NULL (x86_pmu_late_setup, *x86_pmu.late_setup); +/* * This one is magic, it will get called even when PMU init fails (because * there is no PMU), in which case it should simply return NULL.
@@ -1298,6 +1300,15 @@ static void x86_pmu_enable(struct pmu *pmu)if (cpuc->n_added) {int n_running = cpuc->n_events - cpuc->n_added; + +/* +* The late setup (after counters are scheduled) +* is required for some cases, e.g., PEBS counters +* snapshotting. Because an accurate counter index +* is needed. +*/ +static_call_cond (x86_pmu_late_setup)(); +/* * apply assignment obtained either from * hw_perf_group_sched_in() or x86_pmu_enable()
@@ -2035,6 +2046,8 @@ static void x86_pmu_static_call_update(void)static_call_update (x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);static_call_update (x86_pmu_filter, x86_pmu.filter); + +static_call_update (x86_pmu_late_setup, x86_pmu.late_setup); }static void _x86_pmu_read (struct perf_event *event)
arch/x86/events/intel/core.c +60 -15
131 lines not on this page · raw message
arch/x86/events/intel/ds.c +182 -9
242 lines not on this page · raw message
arch/x86/events/perf_event.h +13 -0
@@ -115,6 +115,11 @@ static inline bool is_branch_counters_group(struct perf_event *event)return event->group_leader->hw.flags &PERF_X86_EVENT_BRANCH_COUNTERS ; } +static inline bool is_pebs_counter_event_group (struct perf_event *event) +{ +return event->group_leader->hw.flags &PERF_X86_EVENT_PEBS_CNTR ; +} +struct amd_nb {int nb_id;/* NorthBridge id */ int refcnt;/* reference count */
@@ -800,6 +805,7 @@ struct x86_pmu {u64 (*update)(struct perf_event *event);int (*hw_config)(struct perf_event *event);int (*schedule_events)(struct cpu_hw_events *cpuc,int n,int *assign); +void (*late_setup)(void );unsigned eventsel;unsigned perfctr;unsigned fixedctr;
@@ -1108,6 +1114,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update);DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DECLARE_STATIC_CALL (x86_pmu_late_setup, *x86_pmu.late_setup); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {
@@ -1149,6 +1156,12 @@ extern u64 __read_mostly hw_cache_extra_regsu64 x86_perf_event_update (struct perf_event *event); +static inline u64 intel_pmu_topdown_event_update (struct perf_event *event,u64 *val) +{ +return x86_perf_event_update (event); +} +DECLARE_STATIC_CALL (intel_pmu_update_topdown_event, intel_pmu_topdown_event_update); +static inline unsigned int x86_pmu_config_addr (int index) { return x86_pmu.eventsel + (x86_pmu.addr_offset ?
arch/x86/events/perf_event_flags.h +1 -1
@@ -9,7 +9,7 @@ PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */ PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */ PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */ PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */ - /* 0x00080 */ +PERF_ARCH(PEBS_CNTR, 0x00080) /* PEBS counters snapshot */ PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */ PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */PERF_ARCH (LARGE_PEBS ,0x00400 )/* use large PEBS */
arch/x86/include/asm/perf_event.h +15 -0
@@ -141,6 +141,12 @@#define PEBS_DATACFG_XMMS BIT_ULL(2)#define PEBS_DATACFG_LBRS BIT_ULL(3)#define PEBS_DATACFG_LBR_SHIFT 24 +#define PEBS_DATACFG_CNTR BIT_ULL(4) +#define PEBS_DATACFG_CNTR_SHIFT 32 +#define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0) +#define PEBS_DATACFG_FIX_SHIFT 48 +#define PEBS_DATACFG_FIX_MASK GENMASK_ULL(7, 0) +#define PEBS_DATACFG_METRICS BIT_ULL(5)/* Steal the highest bit of pebs_data_cfg for SW usage */ #define PEBS_UPDATE_DS_SW BIT_ULL(63)
@@ -471,6 +477,15 @@ struct pebs_xmm {#define IBS_CPUID_FEATURES 0x8000001b +struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3 +/* * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but * bit 0 is used to indicate the existence of IBS.
Signature · 1 line
2.38.1
Peter Zijlstra
maintainerRe: [PATCH V10 4/4] perf/x86/intel: Support PEBS counters snapshotting
arch/x86/include/asm/perf_event.h struct pebs_xmm { · line 487
Quoted 12 lines · On Tue, Jan 21, 2025 at 07:23:03AM -0800, [email protected] wrote:
arch/x86/include/asm/perf_event.h +8
-0@@ -471,6 +477,15 @@ struct pebs_xmm {#define IBS_CPUID_FEATURES 0x8000001b +struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3
I've moved this up a little, so that all the adaptive pebs crud is together.
Other than that, I've picked up these patches and will merge the lot past -rc1.
Re: [PATCH V10 4/4] perf/x86/intel: Support PEBS counters snapshotting
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · On Tue, Jan 21, 2025 at 07:23:03AM -0800, [email protected] wrote:
arch/x86/include/asm/perf_event.h +0
-0@@ -471,6 +477,15 @@ struct pebs_xmm {#define IBS_CPUID_FEATURES 0x8000001b
Quoted 12 lines
arch/x86/include/asm/perf_event.h struct pebs_xmm { · line 487
Quoted 8 lines
arch/x86/include/asm/perf_event.h +8
-0+struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3I've moved this up a little, so that all the adaptive pebs crud is together.
Sure. Thanks!
Quoted 3 lines
Other than that, I've picked up these patches and will merge the lot past -rc1.
Thanks!
Kan
Re: [PATCH V10 4/4] perf/x86/intel: Support PEBS counters snapshotting
In reply to Peter ZijlstraHi Peter,
Quoted 18 lines · Peter Zijlstra
arch/x86/include/asm/perf_event.h struct pebs_xmm { · line 487
Quoted 12 lines · On Tue, Jan 21, 2025 at 07:23:03AM -0800, [email protected] wrote:
arch/x86/include/asm/perf_event.h +8
-0@@ -471,6 +477,15 @@ struct pebs_xmm {#define IBS_CPUID_FEATURES 0x8000001b +struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3I've moved this up a little, so that all the adaptive pebs crud is together.
Quoted 3 lines
Other than that, I've picked up these patches and will merge the lot past -rc1.
We found a bug when testing with some non-precise events in freq mode. I've submitted a patch to fix it.
https://lore.kernel.org/lkml/[email protected]/
Could you please fold it into the patch ("perf/x86/intel: Support PEBS counters snapshotting")?
Sorry for any inconvenience.
Thanks,
Kan
[tip: perf/core] perf/x86/intel: Support PEBS counters snapshotting
In reply to [email protected]The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 5169dc7a97c195a1291b0101f562a97c11b390e9 Gitweb: https://git.kernel.org/tip/5169dc7a97c195a1291b0101f562a97c11b390e9 Author: Kan Liang <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:03 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 03 Feb 2025 11:46:07 +01:00
perf/x86/intel: Support PEBS counters snapshotting
The counters snapshotting is a new adaptive PEBS extension, which can capture programmable counters, fixed-function counters, and performance metrics in a PEBS record. The feature is available in the PEBS format V6.
The target counters can be configured in the new fields of MSR_PEBS_CFG. Then the PEBS HW will generate the bit mask of counters (Counters Group Header) followed by the content of all the requested counters into a PEBS record.
The current Linux perf sample read feature can read all events in the group when any event in the group is overflowed. But the rdpmc in the NMI/overflow handler has a small gap from overflow. Also, there is some overhead for each rdpmc read. The counters snapshotting feature can be used as an accurate and low-overhead replacement.
Extend intel_update_topdown_event() to accept the value from PEBS records.
Add a new PEBS_CNTR flag to indicate a sample read group that utilizes the counters snapshotting feature. When the group is scheduled, the PEBS configure can be updated accordingly.
To prevent the case that a PEBS record value might be in the past relative to what is already in the event, perf always stops the PMU and drains the PEBS buffer before updating the corresponding event->count.
--- arch/x86/events/core.c | 13 ++- arch/x86/events/intel/core.c | 75 ++++++++--- arch/x86/events/intel/ds.c | 191 ++++++++++++++++++++++++++-- arch/x86/events/perf_event.h | 13 ++- arch/x86/events/perf_event_flags.h | 2 +- arch/x86/include/asm/perf_event.h | 15 ++- 6 files changed, 284 insertions(+), 25 deletions(-)
arch/x86/events/core.c +13 -0
@@ -94,6 +94,8 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);DEFINE_STATIC_CALL_NULL (x86_pmu_filter, *x86_pmu.filter); +DEFINE_STATIC_CALL_NULL (x86_pmu_late_setup, *x86_pmu.late_setup); +/* * This one is magic, it will get called even when PMU init fails (because * there is no PMU), in which case it should simply return NULL.
@@ -1298,6 +1300,15 @@ static void x86_pmu_enable(struct pmu *pmu)if (cpuc->n_added) {int n_running = cpuc->n_events - cpuc->n_added; + +/* +* The late setup (after counters are scheduled) +* is required for some cases, e.g., PEBS counters +* snapshotting. Because an accurate counter index +* is needed. +*/ +static_call_cond (x86_pmu_late_setup)(); +/* * apply assignment obtained either from * hw_perf_group_sched_in() or x86_pmu_enable()
@@ -2035,6 +2046,8 @@ static void x86_pmu_static_call_update(void)static_call_update (x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);static_call_update (x86_pmu_filter, x86_pmu.filter); + +static_call_update (x86_pmu_late_setup, x86_pmu.late_setup); }static void _x86_pmu_read (struct perf_event *event)
arch/x86/events/intel/core.c +60 -15
131 lines not on this page · raw message
arch/x86/events/intel/ds.c +182 -9
242 lines not on this page · raw message
arch/x86/events/perf_event.h +13 -0
@@ -115,6 +115,11 @@ static inline bool is_branch_counters_group(struct perf_event *event)return event->group_leader->hw.flags &PERF_X86_EVENT_BRANCH_COUNTERS ; } +static inline bool is_pebs_counter_event_group (struct perf_event *event) +{ +return event->group_leader->hw.flags &PERF_X86_EVENT_PEBS_CNTR ; +} +struct amd_nb {int nb_id;/* NorthBridge id */ int refcnt;/* reference count */
@@ -800,6 +805,7 @@ struct x86_pmu {u64 (*update)(struct perf_event *event);int (*hw_config)(struct perf_event *event);int (*schedule_events)(struct cpu_hw_events *cpuc,int n,int *assign); +void (*late_setup)(void );unsigned eventsel;unsigned perfctr;unsigned fixedctr;
@@ -1108,6 +1114,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update);DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DECLARE_STATIC_CALL (x86_pmu_late_setup, *x86_pmu.late_setup); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {
@@ -1149,6 +1156,12 @@ extern u64 __read_mostly hw_cache_extra_regsu64 x86_perf_event_update (struct perf_event *event); +static inline u64 intel_pmu_topdown_event_update (struct perf_event *event,u64 *val) +{ +return x86_perf_event_update (event); +} +DECLARE_STATIC_CALL (intel_pmu_update_topdown_event, intel_pmu_topdown_event_update); +static inline unsigned int x86_pmu_config_addr (int index) { return x86_pmu.eventsel + (x86_pmu.addr_offset ?
arch/x86/events/perf_event_flags.h +1 -1
@@ -9,7 +9,7 @@ PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */ PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */ PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */ PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */ - /* 0x00080 */ +PERF_ARCH(PEBS_CNTR, 0x00080) /* PEBS counters snapshot */ PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */ PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */PERF_ARCH (LARGE_PEBS ,0x00400 )/* use large PEBS */
arch/x86/include/asm/perf_event.h +15 -0
@@ -141,6 +141,12 @@#define PEBS_DATACFG_XMMS BIT_ULL(2)#define PEBS_DATACFG_LBRS BIT_ULL(3)#define PEBS_DATACFG_LBR_SHIFT 24 +#define PEBS_DATACFG_CNTR BIT_ULL(4) +#define PEBS_DATACFG_CNTR_SHIFT 32 +#define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0) +#define PEBS_DATACFG_FIX_SHIFT 48 +#define PEBS_DATACFG_FIX_MASK GENMASK_ULL(7, 0) +#define PEBS_DATACFG_METRICS BIT_ULL(5)/* Steal the highest bit of pebs_data_cfg for SW usage */ #define PEBS_UPDATE_DS_SW BIT_ULL(63)
@@ -460,6 +466,15 @@ struct pebs_xmm {u64 xmm[16 *2 ];/* two entries for each register */ }; +struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3 +/* * AMD Extended Performance Monitoring and Debug cpuid feature detection */
[tip: perf/core] perf/x86/intel: Support PEBS counters snapshotting
In reply to [email protected]The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: e02e9b0374c378aab016ae8ace60d9d98ab8caa6 Gitweb: https://git.kernel.org/tip/e02e9b0374c378aab016ae8ace60d9d98ab8caa6 Author: Kan Liang <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:03 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 05 Feb 2025 10:29:45 +01:00
perf/x86/intel: Support PEBS counters snapshotting
The counters snapshotting is a new adaptive PEBS extension, which can capture programmable counters, fixed-function counters, and performance metrics in a PEBS record. The feature is available in the PEBS format V6.
The target counters can be configured in the new fields of MSR_PEBS_CFG. Then the PEBS HW will generate the bit mask of counters (Counters Group Header) followed by the content of all the requested counters into a PEBS record.
The current Linux perf sample read feature can read all events in the group when any event in the group is overflowed. But the rdpmc in the NMI/overflow handler has a small gap from overflow. Also, there is some overhead for each rdpmc read. The counters snapshotting feature can be used as an accurate and low-overhead replacement.
Extend intel_update_topdown_event() to accept the value from PEBS records.
Add a new PEBS_CNTR flag to indicate a sample read group that utilizes the counters snapshotting feature. When the group is scheduled, the PEBS configure can be updated accordingly.
To prevent the case that a PEBS record value might be in the past relative to what is already in the event, perf always stops the PMU and drains the PEBS buffer before updating the corresponding event->count.
--- arch/x86/events/core.c | 13 ++- arch/x86/events/intel/core.c | 75 ++++++++--- arch/x86/events/intel/ds.c | 191 ++++++++++++++++++++++++++-- arch/x86/events/perf_event.h | 13 ++- arch/x86/events/perf_event_flags.h | 2 +- arch/x86/include/asm/perf_event.h | 15 ++- 6 files changed, 284 insertions(+), 25 deletions(-)
arch/x86/events/core.c +13 -0
@@ -94,6 +94,8 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);DEFINE_STATIC_CALL_NULL (x86_pmu_filter, *x86_pmu.filter); +DEFINE_STATIC_CALL_NULL (x86_pmu_late_setup, *x86_pmu.late_setup); +/* * This one is magic, it will get called even when PMU init fails (because * there is no PMU), in which case it should simply return NULL.
@@ -1298,6 +1300,15 @@ static void x86_pmu_enable(struct pmu *pmu)if (cpuc->n_added) {int n_running = cpuc->n_events - cpuc->n_added; + +/* +* The late setup (after counters are scheduled) +* is required for some cases, e.g., PEBS counters +* snapshotting. Because an accurate counter index +* is needed. +*/ +static_call_cond (x86_pmu_late_setup)(); +/* * apply assignment obtained either from * hw_perf_group_sched_in() or x86_pmu_enable()
@@ -2035,6 +2046,8 @@ static void x86_pmu_static_call_update(void)static_call_update (x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);static_call_update (x86_pmu_filter, x86_pmu.filter); + +static_call_update (x86_pmu_late_setup, x86_pmu.late_setup); }static void _x86_pmu_read (struct perf_event *event)
arch/x86/events/intel/core.c +60 -15
131 lines not on this page · raw message
arch/x86/events/intel/ds.c +182 -9
242 lines not on this page · raw message
arch/x86/events/perf_event.h +13 -0
@@ -115,6 +115,11 @@ static inline bool is_branch_counters_group(struct perf_event *event)return event->group_leader->hw.flags &PERF_X86_EVENT_BRANCH_COUNTERS ; } +static inline bool is_pebs_counter_event_group (struct perf_event *event) +{ +return event->group_leader->hw.flags &PERF_X86_EVENT_PEBS_CNTR ; +} +struct amd_nb {int nb_id;/* NorthBridge id */ int refcnt;/* reference count */
@@ -800,6 +805,7 @@ struct x86_pmu {u64 (*update)(struct perf_event *event);int (*hw_config)(struct perf_event *event);int (*schedule_events)(struct cpu_hw_events *cpuc,int n,int *assign); +void (*late_setup)(void );unsigned eventsel;unsigned perfctr;unsigned fixedctr;
@@ -1108,6 +1114,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update);DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DECLARE_STATIC_CALL (x86_pmu_late_setup, *x86_pmu.late_setup); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {
@@ -1149,6 +1156,12 @@ extern u64 __read_mostly hw_cache_extra_regsu64 x86_perf_event_update (struct perf_event *event); +static inline u64 intel_pmu_topdown_event_update (struct perf_event *event,u64 *val) +{ +return x86_perf_event_update (event); +} +DECLARE_STATIC_CALL (intel_pmu_update_topdown_event, intel_pmu_topdown_event_update); +static inline unsigned int x86_pmu_config_addr (int index) { return x86_pmu.eventsel + (x86_pmu.addr_offset ?
arch/x86/events/perf_event_flags.h +1 -1
@@ -9,7 +9,7 @@ PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */ PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */ PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */ PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */ - /* 0x00080 */ +PERF_ARCH(PEBS_CNTR, 0x00080) /* PEBS counters snapshot */ PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */ PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */PERF_ARCH (LARGE_PEBS ,0x00400 )/* use large PEBS */
arch/x86/include/asm/perf_event.h +15 -0
@@ -141,6 +141,12 @@#define PEBS_DATACFG_XMMS BIT_ULL(2)#define PEBS_DATACFG_LBRS BIT_ULL(3)#define PEBS_DATACFG_LBR_SHIFT 24 +#define PEBS_DATACFG_CNTR BIT_ULL(4) +#define PEBS_DATACFG_CNTR_SHIFT 32 +#define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0) +#define PEBS_DATACFG_FIX_SHIFT 48 +#define PEBS_DATACFG_FIX_MASK GENMASK_ULL(7, 0) +#define PEBS_DATACFG_METRICS BIT_ULL(5)/* Steal the highest bit of pebs_data_cfg for SW usage */ #define PEBS_UPDATE_DS_SW BIT_ULL(63)
@@ -460,6 +466,15 @@ struct pebs_xmm {u64 xmm[16 *2 ];/* two entries for each register */ }; +struct pebs_cntr_header { +u32 cntr; +u32 fixed; +u32 metrics; +u32 reserved; +}; + +#define INTEL_CNTR_METRICS 0x3 +/* * AMD Extended Performance Monitoring and Debug cpuid feature detection */
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: da6e8c7a252bb0bbad53f1e6993460e17d891652 Gitweb: https://git.kernel.org/tip/da6e8c7a252bb0bbad53f1e6993460e17d891652 Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:00 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 03 Feb 2025 11:46:07 +01:00
perf/x86/intel: Apply static call for drain_pebs
The x86_pmu_drain_pebs static call was introduced in commit 7c9903c9bf71 ("x86/perf, static_call: Optimize x86_pmu methods"), but it's not really used to replace the old method.
Apply the static call for drain_pebs.
--- arch/x86/events/intel/core.c | 2 +- arch/x86/events/intel/ds.c | 2 +- arch/x86/events/perf_event.h | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -3076,7 +3076,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data); - x86_pmu.drain_pebs (regs, &data); +static_call (x86_pmu_drain_pebs)(regs, &data); status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/*
arch/x86/events/intel/ds.c +1 -1
@@ -957,7 +957,7 @@ static inline void intel_pmu_drain_pebs_buffer(void) {struct perf_sample_data data; - x86_pmu.drain_pebs (NULL , &data); +static_call (x86_pmu_drain_pebs)(NULL , &data); }/*
arch/x86/events/perf_event.h +1 -0
@@ -1107,6 +1107,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update); +DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 314dfe10576912e1d786b13c5d4eee8c51b63caa Gitweb: https://git.kernel.org/tip/314dfe10576912e1d786b13c5d4eee8c51b63caa Author: Peter Zijlstra (Intel) <[email protected]> AuthorDate: Tue, 21 Jan 2025 07:23:00 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 05 Feb 2025 10:29:45 +01:00
perf/x86/intel: Apply static call for drain_pebs
The x86_pmu_drain_pebs static call was introduced in commit 7c9903c9bf71 ("x86/perf, static_call: Optimize x86_pmu methods"), but it's not really used to replace the old method.
Apply the static call for drain_pebs.
--- arch/x86/events/intel/core.c | 2 +- arch/x86/events/intel/ds.c | 2 +- arch/x86/events/perf_event.h | 1 + 3 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -3076,7 +3076,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++;x86_pmu_handle_guest_pebs (regs, &data); - x86_pmu.drain_pebs (regs, &data); +static_call (x86_pmu_drain_pebs)(regs, &data); status &= intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/*
arch/x86/events/intel/ds.c +1 -1
@@ -957,7 +957,7 @@ static inline void intel_pmu_drain_pebs_buffer(void) {struct perf_sample_data data; - x86_pmu.drain_pebs (NULL , &data); +static_call (x86_pmu_drain_pebs)(NULL , &data); }/*
arch/x86/events/perf_event.h +1 -0
@@ -1107,6 +1107,7 @@ extern struct x86_pmu x86_pmu __read_mostly;DECLARE_STATIC_CALL (x86_pmu_set_period, *x86_pmu.set_period);DECLARE_STATIC_CALL (x86_pmu_update, *x86_pmu.update); +DECLARE_STATIC_CALL (x86_pmu_drain_pebs, *x86_pmu.drain_pebs); static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) {