perf: Reduce stack usage (and misc bits)
Subsystems (9)
Peter Zijlstra
maintainercover perf: Reduce stack usage (and misc bits)
Hi,
Steve reported an NMI stack overflow when running perf and function tracing together. Thomas found that we had 4 copies of struct perf_sample_data on-stack.
These here patches reduce that to 2 copies and half the size of it.
So just for perf_sample_data we go from 4*384=1536 to 2*192=384 bytes of stack. Also remove one struct pt_regs instance from __intel_pmu_pebs_event(); it has another instance in struct x86_perf_regs which I haven't yet managed to offload.
Perf seems to still work... :-)
Peter Zijlstra
maintainerPATCH 1/6 perf: Reduce stack usage of perf_output_begin()
__perf_output_begin() has an on-stack struct perf_sample_data in the unlikely case it needs to generate a LOST record. However, every call to perf_output_begin() must already have a perf_sample_data on-stack.
--- arch/powerpc/perf/imc-pmu.c | 2 +- arch/s390/kernel/perf_cpum_sf.c | 2 +- arch/x86/events/intel/ds.c | 4 ++-- include/linux/perf_event.h | 7 +++++-- kernel/events/core.c | 32 +++++++++++++++++--------------- kernel/events/ring_buffer.c | 20 +++++++++++--------- 6 files changed, 37 insertions(+), 30 deletions(-)
arch/powerpc/perf/imc-pmu.c +1 -1
@@ -1336,7 +1336,7 @@ static void dump_trace_imc_data(struct p/* If this is a valid record, create the sample */ struct perf_output_handle handle; -if (perf_output_begin (&handle, event, header.size)) +if (perf_output_begin (&handle, &data, event, header.size))return ;perf_output_sample (&handle, &header, &data, event);
arch/s390/kernel/perf_cpum_sf.c +1 -1
@@ -672,7 +672,7 @@ static void cpumsf_output_event_pid(strurcu_read_lock ();perf_prepare_sample (&header, data, event, regs); -if (perf_output_begin (&handle, event, header.size)) +if (perf_output_begin (&handle, data, event, header.size)) goto out;/* Update the process ID (see also kernel/events/core.c) */
arch/x86/events/intel/ds.c +2 -2
@@ -642,8 +642,8 @@ int intel_pmu_drain_bts_buffer(void)rcu_read_lock ();perf_prepare_sample (&header, &data, event, ®s); -if (perf_output_begin (&handle, event, header.size * - (top - base - skip))) +if (perf_output_begin (&handle, &data, event, + header.size * (top - base - skip))) goto unlock;for (at = base; at < top; at++) {
include/linux/perf_event.h +5 -2
@@ -1400,11 +1400,14 @@ perf_event_addr_filters(struct perf_evenextern void perf_event_addr_filters_sync (struct perf_event *event);extern int perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size);extern int perf_output_begin_forward (struct perf_output_handle *handle, -struct perf_event *event, -unsigned int size); +struct perf_sample_data *data, +struct perf_event *event, +unsigned int size);extern int perf_output_begin_backward (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size);
kernel/events/core.c +17 -15
@@ -7186,6 +7186,7 @@ __perf_event_output(struct perf_event *estruct perf_sample_data *data, struct pt_regs *regs, int (*output_begin)(struct perf_output_handle *, +struct perf_sample_data *,struct perf_event *, unsigned int)) {
@@ -7198,7 +7199,7 @@ __perf_event_output(struct perf_event *eperf_prepare_sample (&header, data, event, regs); - err =output_begin (&handle, event, header.size); + err =output_begin (&handle, data, event, header.size);if (err) goto exit;
@@ -7264,7 +7265,7 @@ perf_event_read_event(struct perf_eventint ret;perf_event_header__init_id (&read_event.header, &sample, event); - ret =perf_output_begin (&handle, event, read_event.header.size); + ret =perf_output_begin (&handle, &sample, event, read_event.header.size);if (ret)return ;
@@ -7533,7 +7534,7 @@ static void perf_event_task_output(strucperf_event_header__init_id (&task_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, task_event->event_id.header.size);if (ret) goto out;
@@ -7636,7 +7637,7 @@ static void perf_event_comm_output(strucreturn ;perf_event_header__init_id (&comm_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, comm_event->event_id.header.size);if (ret)
@@ -7736,7 +7737,7 @@ static void perf_event_namespaces_outputperf_event_header__init_id (&namespaces_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, namespaces_event->event_id.header.size);if (ret) goto out;
@@ -7863,7 +7864,7 @@ static void perf_event_cgroup_output(strperf_event_header__init_id (&cgroup_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, cgroup_event->event_id.header.size);if (ret) goto out;
@@ -7989,7 +7990,7 @@ static void perf_event_mmap_output(struc }perf_event_header__init_id (&mmap_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, mmap_event->event_id.header.size);if (ret) goto out;
@@ -8299,7 +8300,7 @@ void perf_event_aux_event(struct perf_evint ret;perf_event_header__init_id (&rec.header, &sample, event); - ret =perf_output_begin (&handle, event, rec.header.size); + ret =perf_output_begin (&handle, &sample, event, rec.header.size);if (ret)return ;
@@ -8333,7 +8334,7 @@ void perf_log_lost_samples(struct perf_eperf_event_header__init_id (&lost_samples_event.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, lost_samples_event.header.size);if (ret)return ;
@@ -8388,7 +8389,7 @@ static void perf_event_switch_output(strperf_event_header__init_id (&se->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, se->event_id.header.size); + ret =perf_output_begin (&handle, &sample, event, se->event_id.header.size);if (ret)return ;
@@ -8463,7 +8464,7 @@ static void perf_log_throttle(struct perperf_event_header__init_id (&throttle_event.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, throttle_event.header.size);if (ret)return ;
@@ -8506,7 +8507,7 @@ static void perf_event_ksymbol_output(stperf_event_header__init_id (&ksymbol_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, ksymbol_event->event_id.header.size);if (ret)return ;
@@ -8596,7 +8597,7 @@ static void perf_event_bpf_output(structperf_event_header__init_id (&bpf_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, data, event, bpf_event->event_id.header.size);if (ret)return ;
@@ -8705,7 +8706,8 @@ static void perf_event_text_poke_output(perf_event_header__init_id (&text_poke_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, text_poke_event->event_id.header.size); + ret =perf_output_begin (&handle, &sample, event, + text_poke_event->event_id.header.size);if (ret)return ;
@@ -8786,7 +8788,7 @@ static void perf_log_itrace_start(struct rec.tid =perf_event_tid (event, current);perf_event_header__init_id (&rec.header, &sample, event); - ret =perf_output_begin (&handle, event, rec.header.size); + ret =perf_output_begin (&handle, &sample, event, rec.header.size);if (ret)return ;
kernel/events/ring_buffer.c +11 -9
@@ -147,6 +147,7 @@ ring_buffer_has_space(unsigned long head static __always_inline int__perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size,bool backward) {
@@ -237,18 +238,16 @@ __perf_output_begin(struct perf_output_h handle->size = (1UL << page_shift) - offset;if (unlikely (have_lost)) { -struct perf_sample_data sample_data; - lost_event.header.size =sizeof (lost_event); lost_event.header.type =PERF_RECORD_LOST ; lost_event.header.misc =0 ; lost_event.id = event->id; lost_event.lost =local_xchg (&rb->lost,0 ); -perf_event_header__init_id (&lost_event.header, - &sample_data, event); +/* XXX mostly redundant; @data is already fully initializes */ +perf_event_header__init_id (&lost_event.header, data, event);perf_output_put (handle, lost_event); -perf_event__output_id_sample (event, handle, &sample_data); +perf_event__output_id_sample (event, handle, data); }return 0 ;
@@ -263,22 +262,25 @@ __perf_output_begin(struct perf_output_h }int perf_output_begin_forward (struct perf_output_handle *handle, -struct perf_event *event,unsigned int size) +struct perf_sample_data *data, +struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, false); +return __perf_output_begin (handle, data, event, size, false); }int perf_output_begin_backward (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, true); +return __perf_output_begin (handle, data, event, size, true); }int perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, +return __perf_output_begin (handle, data, event, size,unlikely (is_write_backward (event))); }
[tip: perf/urgent] perf: Reduce stack usage of perf_output_begin()
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: 267fb27352b6fc9fdbad753127a239f75618ecbc Gitweb: https://git.kernel.org/tip/267fb27352b6fc9fdbad753127a239f75618ecbc Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 15:50:32 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:33 +01:00
perf: Reduce stack usage of perf_output_begin()
__perf_output_begin() has an on-stack struct perf_sample_data in the unlikely case it needs to generate a LOST record. However, every call to perf_output_begin() must already have a perf_sample_data on-stack.
--- arch/powerpc/perf/imc-pmu.c | 2 +- arch/s390/kernel/perf_cpum_sf.c | 2 +- arch/x86/events/intel/ds.c | 4 ++-- include/linux/perf_event.h | 7 +++++-- kernel/events/core.c | 32 +++++++++++++++++--------------- kernel/events/ring_buffer.c | 20 +++++++++++--------- 6 files changed, 37 insertions(+), 30 deletions(-)
arch/powerpc/perf/imc-pmu.c +1 -1
@@ -1336,7 +1336,7 @@ static void dump_trace_imc_data(struct perf_event *event)/* If this is a valid record, create the sample */ struct perf_output_handle handle; -if (perf_output_begin (&handle, event, header.size)) +if (perf_output_begin (&handle, &data, event, header.size))return ;perf_output_sample (&handle, &header, &data, event);
arch/s390/kernel/perf_cpum_sf.c +1 -1
@@ -672,7 +672,7 @@ static void cpumsf_output_event_pid(struct perf_event *event,rcu_read_lock ();perf_prepare_sample (&header, data, event, regs); -if (perf_output_begin (&handle, event, header.size)) +if (perf_output_begin (&handle, data, event, header.size)) goto out;/* Update the process ID (see also kernel/events/core.c) */
arch/x86/events/intel/ds.c +2 -2
@@ -642,8 +642,8 @@ int intel_pmu_drain_bts_buffer(void)rcu_read_lock ();perf_prepare_sample (&header, &data, event, ®s); -if (perf_output_begin (&handle, event, header.size * - (top - base - skip))) +if (perf_output_begin (&handle, &data, event, + header.size * (top - base - skip))) goto unlock;for (at = base; at < top; at++) {
include/linux/perf_event.h +5 -2
@@ -1400,11 +1400,14 @@ perf_event_addr_filters(struct perf_event *event)extern void perf_event_addr_filters_sync (struct perf_event *event);extern int perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size);extern int perf_output_begin_forward (struct perf_output_handle *handle, -struct perf_event *event, -unsigned int size); +struct perf_sample_data *data, +struct perf_event *event, +unsigned int size);extern int perf_output_begin_backward (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size);
kernel/events/core.c +17 -15
@@ -7186,6 +7186,7 @@ __perf_event_output(struct perf_event *event,struct perf_sample_data *data, struct pt_regs *regs, int (*output_begin)(struct perf_output_handle *, +struct perf_sample_data *,struct perf_event *, unsigned int)) {
@@ -7198,7 +7199,7 @@ __perf_event_output(struct perf_event *event,perf_prepare_sample (&header, data, event, regs); - err =output_begin (&handle, event, header.size); + err =output_begin (&handle, data, event, header.size);if (err) goto exit;
@@ -7264,7 +7265,7 @@ perf_event_read_event(struct perf_event *event,int ret;perf_event_header__init_id (&read_event.header, &sample, event); - ret =perf_output_begin (&handle, event, read_event.header.size); + ret =perf_output_begin (&handle, &sample, event, read_event.header.size);if (ret)return ;
@@ -7533,7 +7534,7 @@ static void perf_event_task_output(struct perf_event *event,perf_event_header__init_id (&task_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, task_event->event_id.header.size);if (ret) goto out;
@@ -7636,7 +7637,7 @@ static void perf_event_comm_output(struct perf_event *event,return ;perf_event_header__init_id (&comm_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, comm_event->event_id.header.size);if (ret)
@@ -7736,7 +7737,7 @@ static void perf_event_namespaces_output(struct perf_event *event,perf_event_header__init_id (&namespaces_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, namespaces_event->event_id.header.size);if (ret) goto out;
@@ -7863,7 +7864,7 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data)perf_event_header__init_id (&cgroup_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, cgroup_event->event_id.header.size);if (ret) goto out;
@@ -7989,7 +7990,7 @@ static void perf_event_mmap_output(struct perf_event *event, }perf_event_header__init_id (&mmap_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, mmap_event->event_id.header.size);if (ret) goto out;
@@ -8299,7 +8300,7 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head,int ret;perf_event_header__init_id (&rec.header, &sample, event); - ret =perf_output_begin (&handle, event, rec.header.size); + ret =perf_output_begin (&handle, &sample, event, rec.header.size);if (ret)return ;
@@ -8333,7 +8334,7 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost)perf_event_header__init_id (&lost_samples_event.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, lost_samples_event.header.size);if (ret)return ;
@@ -8388,7 +8389,7 @@ static void perf_event_switch_output(struct perf_event *event, void *data)perf_event_header__init_id (&se->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, se->event_id.header.size); + ret =perf_output_begin (&handle, &sample, event, se->event_id.header.size);if (ret)return ;
@@ -8463,7 +8464,7 @@ static void perf_log_throttle(struct perf_event *event, int enable)perf_event_header__init_id (&throttle_event.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, throttle_event.header.size);if (ret)return ;
@@ -8506,7 +8507,7 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data)perf_event_header__init_id (&ksymbol_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, &sample, event, ksymbol_event->event_id.header.size);if (ret)return ;
@@ -8596,7 +8597,7 @@ static void perf_event_bpf_output(struct perf_event *event, void *data)perf_event_header__init_id (&bpf_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, + ret =perf_output_begin (&handle, data, event, bpf_event->event_id.header.size);if (ret)return ;
@@ -8705,7 +8706,8 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data)perf_event_header__init_id (&text_poke_event->event_id.header, &sample, event); - ret =perf_output_begin (&handle, event, text_poke_event->event_id.header.size); + ret =perf_output_begin (&handle, &sample, event, + text_poke_event->event_id.header.size);if (ret)return ;
@@ -8786,7 +8788,7 @@ static void perf_log_itrace_start(struct perf_event *event) rec.tid =perf_event_tid (event, current);perf_event_header__init_id (&rec.header, &sample, event); - ret =perf_output_begin (&handle, event, rec.header.size); + ret =perf_output_begin (&handle, &sample, event, rec.header.size);if (ret)return ;
kernel/events/ring_buffer.c +11 -9
@@ -147,6 +147,7 @@ ring_buffer_has_space(unsigned long head, unsigned long tail, static __always_inline int__perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size,bool backward) {
@@ -237,18 +238,16 @@ __perf_output_begin(struct perf_output_handle *handle, handle->size = (1UL << page_shift) - offset;if (unlikely (have_lost)) { -struct perf_sample_data sample_data; - lost_event.header.size =sizeof (lost_event); lost_event.header.type =PERF_RECORD_LOST ; lost_event.header.misc =0 ; lost_event.id = event->id; lost_event.lost =local_xchg (&rb->lost,0 ); -perf_event_header__init_id (&lost_event.header, - &sample_data, event); +/* XXX mostly redundant; @data is already fully initializes */ +perf_event_header__init_id (&lost_event.header, data, event);perf_output_put (handle, lost_event); -perf_event__output_id_sample (event, handle, &sample_data); +perf_event__output_id_sample (event, handle, data); }return 0 ;
@@ -263,22 +262,25 @@ out: }int perf_output_begin_forward (struct perf_output_handle *handle, -struct perf_event *event,unsigned int size) +struct perf_sample_data *data, +struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, false); +return __perf_output_begin (handle, data, event, size, false); }int perf_output_begin_backward (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, true); +return __perf_output_begin (handle, data, event, size, true); }int perf_output_begin (struct perf_output_handle *handle, +struct perf_sample_data *data,struct perf_event *event,unsigned int size) { -return __perf_output_begin (handle, event, size, +return __perf_output_begin (handle, data, event, size,unlikely (is_write_backward (event))); }
Peter Zijlstra
maintainerPATCH 2/6 perf/x86: Reduce stack usage for x86_pmu::drain_pebs()
In reply to Peter Zijlstraintel_pmu_drain_pebs_*() is typically called from handle_pmi_common(), both have an on-stack struct perf_sample_data, which is *big*. Rewire things so that drain_pebs() can use the one handle_pmi_common() has.
--- arch/x86/events/intel/core.c | 2 - arch/x86/events/intel/ds.c | 47 ++++++++++++++++++++++--------------------- arch/x86/events/perf_event.h | 2 - 3 files changed, 27 insertions(+), 24 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -2630,7 +2630,7 @@ static int handle_pmi_common(struct pt_ru64 pebs_enabled = cpuc->pebs_enabled; handled++; - x86_pmu.drain_pebs (regs); + x86_pmu.drain_pebs (regs, &data); status &= x86_pmu.intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/*
arch/x86/events/intel/ds.c +25 -22
118 lines not on this page · raw message
arch/x86/events/perf_event.h +1 -1
@@ -727,7 +727,7 @@ struct x86_pmu {int pebs_record_size;int pebs_buffer_size;int max_pebs_events; -void (*drain_pebs)(struct pt_regs *regs); +void (*drain_pebs)(struct pt_regs *regs,struct perf_sample_data *data);struct event_constraint *pebs_constraints;void (*pebs_aliases)(struct perf_event *event);unsigned long large_pebs_flags;
[tip: perf/urgent] perf/x86: Reduce stack usage for x86_pmu::drain_pebs()
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: 9dfa9a5c9bae3417b87824e7ac73b00c10b6a874 Gitweb: https://git.kernel.org/tip/9dfa9a5c9bae3417b87824e7ac73b00c10b6a874 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 14:58:48 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:33 +01:00
perf/x86: Reduce stack usage for x86_pmu::drain_pebs()
intel_pmu_drain_pebs_*() is typically called from handle_pmi_common(), both have an on-stack struct perf_sample_data, which is *big*. Rewire things so that drain_pebs() can use the one handle_pmi_common() has.
--- arch/x86/events/intel/core.c | 2 +- arch/x86/events/intel/ds.c | 47 ++++++++++++++++++----------------- arch/x86/events/perf_event.h | 2 +- 3 files changed, 27 insertions(+), 24 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -2630,7 +2630,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)u64 pebs_enabled = cpuc->pebs_enabled; handled++; - x86_pmu.drain_pebs (regs); + x86_pmu.drain_pebs (regs, &data); status &= x86_pmu.intel_ctrl |GLOBAL_STATUS_TRACE_TOPAPMI ;/*
arch/x86/events/intel/ds.c +25 -22
118 lines not on this page · raw message
arch/x86/events/perf_event.h +1 -1
@@ -727,7 +727,7 @@ struct x86_pmu {int pebs_record_size;int pebs_buffer_size;int max_pebs_events; -void (*drain_pebs)(struct pt_regs *regs); +void (*drain_pebs)(struct pt_regs *regs,struct perf_sample_data *data);struct event_constraint *pebs_constraints;void (*pebs_aliases)(struct perf_event *event);unsigned long large_pebs_flags;
Peter Zijlstra
maintainerPATCH 3/6 perf: Fix get_recursion_context()
In reply to Peter ZijlstraOne should use in_server_softirq() to detector SoftIRQ context.
--- kernel/events/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/events/internal.h +1 -1
@@ -211,7 +211,7 @@ static inline int get_recursion_context( rctx =3 ; else if (in_irq()) rctx =2 ; - else if (in_softirq()) + else if (in_serving_softirq()) rctx =1 ; else rctx =0 ;
[tip: perf/urgent] perf: Fix get_recursion_context()
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: ce0f17fc93f63ee91428af10b7b2ddef38cd19e5 Gitweb: https://git.kernel.org/tip/ce0f17fc93f63ee91428af10b7b2ddef38cd19e5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 12:49:45 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00
perf: Fix get_recursion_context()
One should use in_serving_softirq() to detect SoftIRQ context.
--- kernel/events/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/events/internal.h +1 -1
@@ -211,7 +211,7 @@ static inline int get_recursion_context(int *recursion) rctx =3 ; else if (in_irq()) rctx =2 ; - else if (in_softirq()) + else if (in_serving_softirq()) rctx =1 ; else rctx =0 ;
Peter Zijlstra
maintainerPATCH 4/6 perf: Optimize get_recursion_context()
In reply to Peter Zijlstra"Look ma, no branches!"
--- kernel/events/internal.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/events/internal.h +8 -9
@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, astatic inline int get_recursion_context (int *recursion) { -int rctx; +unsigned int pc =preempt_count (); +unsigned int rctx =0 ; -if (unlikely (in_nmi ())) - rctx =3 ; -else if (in_irq ()) - rctx =2 ; -else if (in_serving_softirq ()) - rctx =1 ; -else - rctx =0 ; +if (pc & (NMI_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK |SOFTIRQ_OFFSET )) + rctx++;if (recursion[rctx])return -1 ;
Re: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 6 lines · On Fri, 30 Oct 2020 16:13:49 +0100 Peter Zijlstra <[email protected]> wrote:
"Look ma, no branches!"
Cc Jesper Dangaard Brouer <[email protected]>Cc Steven Rostedt <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
Cool trick! :-)
Quoted 32 lines
kernel/events/internal.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)kernel/events/internal.h +8
-9@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, astatic inline int get_recursion_context (int *recursion) { -int rctx; +unsigned int pc =preempt_count (); +unsigned int rctx =0 ; -if (unlikely (in_nmi ())) - rctx =3 ; -else if (in_irq ()) - rctx =2 ; -else if (in_serving_softirq ()) - rctx =1 ; -else - rctx =0 ; +if (pc & (NMI_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK |SOFTIRQ_OFFSET )) + rctx++;if (recursion[rctx])return -1 ;
Signature · 4 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat LinkedIn: http://www.linkedin.com/in/brouer
Re: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 42 lines · On Fri, 30 Oct 2020 18:11:38 +0100 Jesper Dangaard Brouer <[email protected]>…
Quoted 6 lines · On Fri, 30 Oct 2020 16:13:49 +0100 Peter Zijlstra <[email protected]> wrote:
"Look ma, no branches!"
Cc Jesper Dangaard Brouer <[email protected]>Cc Steven Rostedt <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
Cool trick! :-)
Quoted 28 lines
kernel/events/internal.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)kernel/events/internal.h +8
-9@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, astatic inline int get_recursion_context (int *recursion) { -int rctx; +unsigned int pc =preempt_count (); +unsigned int rctx =0 ; -if (unlikely (in_nmi ())) - rctx =3 ; -else if (in_irq ()) - rctx =2 ; -else if (in_serving_softirq ()) - rctx =1 ; -else - rctx =0 ; +if (pc & (NMI_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK )) + rctx++; +if (pc & (NMI_MASK |HARDIRQ_MASK |SOFTIRQ_OFFSET )) + rctx++;
Quoted 4 lines
kernel/events/internal.h DEFINE_OUTPUT_COPY(__output_copy_user, a · line 219
Quoted 4 lines
kernel/events/internal.h +0
-0if (recursion[rctx])return -1 ;
As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get it quickly, and not keep re-implementing it.
/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline int irq_context()
{
unsigned int pc = preempt_count();
int rctx = 0;
if (pc & (NMI_MASK))
rctx++;
if (pc & (NMI_MASK | HARDIRQ_MASK))
rctx++;
if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
rctx++;
return rctx;
}-- Steve
Re: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 2 lines · Steven Rostedt
As this is something that ftrace recursion also does, perhaps we should move this into
interrupt.hso that anyone that needs a counter can get
Not in interrupt.h please. We should create kernel/include/ for stuff which really should only be available in the core kernel code.
Thanks,
tglx
Re: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 7 lines · On Fri, 30 Oct 2020 23:14:18 +0100 Thomas Gleixner <[email protected]> wrote:
Quoted 2 lines · Steven Rostedt
As this is something that ftrace recursion also does, perhaps we should move this into
interrupt.hso that anyone that needs a counter can getNot in
interrupt.hplease. We should createkernel/include/for stuff which really should only be available in the core kernel code.
The recursion protection is needed for anything that registers a callback to ftrace. I have patches that already basically do the same thing (although, with branches) that I'm going to place in include/linux/trace_recursion.h, so that there are helper functions that ftrace callbacks can use, instead of having to implement their own recursion protection. After fixing two bugs in the recursion protection code, it's something that should be reused, instead of everyone making similar mistakes.
Note, I thought that in_nmi() and friends was in interrupt.h, but is really in preempt.h. All the values used in Peter's code is also defined in preempt.h, so why not have something like that there?
I take back adding it to interrupt.h but have it in preempt.h, as it's not defining anything new there.
-- Steve
Peter Zijlstra
maintainerRe: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 6 lines · Steven Rostedt
Note, I thought that
in_nmi()and friends was ininterrupt.h, but is really inpreempt.h. All the values used in Peter's code is also defined inpreempt.h, so why not have something like that there?I take back adding it to
interrupt.hbut have it inpreempt.h, as it's not defining anything new there.
Yeah, preempt.h is the right place for it.
Peter Zijlstra
maintainerRe: [PATCH 4/6] perf: Optimize get_recursion_context()
In reply to Steven RostedtQuoted 3 lines · Steven Rostedt
As this is something that ftrace recursion also does, perhaps we should move this into
interrupt.hso that anyone that needs a counter can get it quickly, and not keep re-implementing it.
Works for me, however:
Quoted 11 lines
/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline intirq_context(){ unsigned int pc = preempt_count(); int rctx = 0;
unsigned
Quoted 10 lines
if (pc & (NMI_MASK)) rctx++; if (pc & (NMI_MASK | HARDIRQ_MASK)) rctx++; if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)) rctx++; return rctx; }
otherwise you'll get an extra instruction to sign extend it, which is daft (yes, i've been staring at GCC output far too much).
Also, gcc-9 does worse (like 1 byte iirc) with:
rctx += !!(pc & (NMI_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
but gcc-10 doesn't seem to care.
RE: [PATCH 4/6] perf: Optimize get_recursion_context()
From: Peter Zijlstra
Quoted 37 lines
Sent: 30 October 2020 23:02
Quoted 3 lines · Steven Rostedt
As this is something that ftrace recursion also does, perhaps we should move this into
interrupt.hso that anyone that needs a counter can get it quickly, and not keep re-implementing it.Works for me, however:
Quoted 11 lines
/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline intirq_context(){ unsigned int pc = preempt_count(); int rctx = 0;unsigned
Quoted 10 lines
if (pc & (NMI_MASK)) rctx++; if (pc & (NMI_MASK | HARDIRQ_MASK)) rctx++; if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)) rctx++; return rctx; }otherwise you'll get an extra instruction to sign extend it, which is daft (yes, i've been staring at GCC output far too much).
Quoted 7 lines
Also, gcc-9 does worse (like 1 byte iirc) with:
rctx += !!(pc & (NMI_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));but gcc-10 doesn't seem to care.
You've made be look at some gcc output (it's raining).
The gcc 7.5.0 I have handy probably generates the best code for:
unsigned char q_2(unsigned int pc)
{
unsigned char rctx = 0;
rctx += !!(pc & (NMI_MASK));
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
return rctx;
}0000000000000000 <q_2>:
0: f7 c7 00 00 f0 00 test $0xf00000,%edi # clock 0 6: 0f 95 c0 setne %al # clock 1 9: f7 c7 00 00 ff 00 test $0xff0000,%edi # clock 0 f: 0f 95 c2 setne %dl # clock 1 12: 01 c2 add %eax,%edx # clock 2 14: 81 e7 00 01 ff 00 and $0xff0100,%edi 1a: 0f 95 c0 setne %al 1d: 01 d0 add %edx,%eax # clock 3 1f: c3 retq
I doubt that is beatable.
I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
RE: [PATCH 4/6] perf: Optimize get_recursion_context()
From: David Laight
Quoted 2 lines
Sent: 31 October 2020 12:12
...
Quoted 24 lines
The gcc 7.5.0 I have handy probably generates the best code for:
unsigned char
q_2(unsigned int pc){ unsigned char rctx = 0; rctx += !!(pc & (NMI_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)); return rctx; }0000000000000000 <
q_2>:0: f7 c7 00 00 f0 00 test $0xf00000,%edi # clock 0 6: 0f 95 c0 setne %al # clock 1 9: f7 c7 00 00 ff 00 test $0xff0000,%edi # clock 0 f: 0f 95 c2 setne %dl # clock 1 12: 01 c2 add %eax,%edx # clock 2 14: 81 e7 00 01 ff 00 and $0xff0100,%edi 1a: 0f 95 c0 setne %al 1d: 01 d0 add %edx,%eax # clock 3 1f: c3 retq
Quoted 1 line
I doubt that is beatable.
I lied, you should be able to get:
test $0xff0000,%edi # clock 0 setne %al # clock 1 test $0xff0100,%edi # clock 0 setne %dl # clock 1 add $fffff000,%edi adc %dl, %al # clock 2
But I suspect getting it from the compiler might be hard!
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Peter Zijlstra
maintainerRe: [PATCH 4/6] perf: Optimize get_recursion_context()
In reply to David LaightQuoted 24 lines · David Laight
The gcc 7.5.0 I have handy probably generates the best code for:
unsigned char
q_2(unsigned int pc){ unsigned char rctx = 0; rctx += !!(pc & (NMI_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)); return rctx; }0000000000000000 <
q_2>:0: f7 c7 00 00 f0 00 test $0xf00000,%edi # clock 0 6: 0f 95 c0 setne %al # clock 1 9: f7 c7 00 00 ff 00 test $0xff0000,%edi # clock 0 f: 0f 95 c2 setne %dl # clock 1 12: 01 c2 add %eax,%edx # clock 2 14: 81 e7 00 01 ff 00 and $0xff0100,%edi 1a: 0f 95 c0 setne %al 1d: 01 d0 add %edx,%eax # clock 3 1f: c3 retq
Quoted 6 lines
I doubt that is beatable.
I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.
Recent GCC just doesn't want to do that. Still, using u8 makes sense, so I've kept that.
RE: [PATCH 4/6] perf: Optimize get_recursion_context()
Quoted 44 lines
-----Original Message-----
From: Peter Zijlstra <[email protected]>
Sent: 09 November 2020 12:13
To: David Laight <[email protected]>
Cc: Steven Rostedt <[email protected]>; Jesper Dangaard Brouer <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH 4/6] perf: Optimizeget_recursion_context()Quoted 24 lines · David Laight
The gcc 7.5.0 I have handy probably generates the best code for:
unsigned char
q_2(unsigned int pc){ unsigned char rctx = 0; rctx += !!(pc & (NMI_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)); return rctx; }0000000000000000 <
q_2>:0: f7 c7 00 00 f0 00 test $0xf00000,%edi # clock 0 6: 0f 95 c0 setne %al # clock 1 9: f7 c7 00 00 ff 00 test $0xff0000,%edi # clock 0 f: 0f 95 c2 setne %dl # clock 1 12: 01 c2 add %eax,%edx # clock 2 14: 81 e7 00 01 ff 00 and $0xff0100,%edi 1a: 0f 95 c0 setne %al 1d: 01 d0 add %edx,%eax # clock 3 1f: c3 retqQuoted 6 lines
I doubt that is beatable.
I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.
Quoted 4 lines
Recent GCC just doesn't want to do that. Still, using u8 makes sense, so I've kept that.
u8 helps x86 because its 'setne' only affects the low 8 bits. I guess that seemed a good idea when it was added (386). It doesn't seem to make the other architectures much worse.
gcc 10.x can be persuaded to generate the above code.
It sometimes seems to me that every new version of gcc is larger, slower and generates worse code than the previous one.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
[tip: perf/urgent] perf: Optimize get_recursion_context()
In reply to Peter ZijlstraThe following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: 09da9c81253dd8e43e0d2d7cea02de6f9f19499d Gitweb: https://git.kernel.org/tip/09da9c81253dd8e43e0d2d7cea02de6f9f19499d Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 13:43:16 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00
perf: Optimize get_recursion_context()
"Look ma, no branches!"
--- kernel/events/internal.h | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-)
kernel/events/internal.h +6 -10
@@ -205,16 +205,12 @@ DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)static inline int get_recursion_context (int *recursion) { -int rctx; - -if (unlikely (in_nmi ())) - rctx =3 ; -else if (in_irq ()) - rctx =2 ; -else if (in_serving_softirq ()) - rctx =1 ; -else - rctx =0 ; +unsigned int pc =preempt_count (); +unsigned char rctx =0 ; + + rctx += !!(pc & (NMI_MASK )); + rctx += !!(pc & (NMI_MASK |HARDIRQ_MASK )); + rctx += !!(pc & (NMI_MASK |HARDIRQ_MASK |SOFTIRQ_OFFSET ));if (recursion[rctx])return -1 ;
Peter Zijlstra
maintainerPATCH 5/6 perf/arch: Remove perf_sample_data::regs_user_copy
In reply to Peter Zijlstrastruct perf_sample_data lives on-stack, we should be careful about it's size. Furthermore, the pt_regs copy in there is only because x86_64 is a trainwreck, solve it differently.
--- arch/arm/kernel/perf_regs.c | 3 +-- arch/arm64/kernel/perf_regs.c | 3 +-- arch/csky/kernel/perf_regs.c | 3 +-- arch/powerpc/perf/perf_regs.c | 3 +-- arch/riscv/kernel/perf_regs.c | 3 +-- arch/s390/kernel/perf_regs.c | 3 +-- arch/x86/kernel/perf_regs.c | 15 +++++++++++---- include/linux/perf_event.h | 6 ------ include/linux/perf_regs.h | 6 ++---- kernel/events/core.c | 8 +++----- 10 files changed, 22 insertions(+), 31 deletions(-)
arch/arm/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/arm64/kernel/perf_regs.c +1 -2
@@ -73,8 +73,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/csky/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/powerpc/perf/perf_regs.c +1 -2
@@ -144,8 +144,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi = (regs_user->regs) ? perf_reg_abi(current) :
arch/riscv/kernel/perf_regs.c +1 -2
@@ -36,8 +36,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/s390/kernel/perf_regs.c +1 -2
@@ -53,8 +53,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) {/* * Use the regs from the first interruption and let
arch/x86/kernel/perf_regs.c +11 -4
@@ -101,8 +101,7 @@ u64 perf_reg_abi(struct task_struct *tas }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
@@ -129,12 +128,20 @@ u64 perf_reg_abi(struct task_struct *tasreturn PERF_SAMPLE_REGS_ABI_64 ; } +static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs); +void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { +struct pt_regs *regs_user_copy =this_cpu_ptr (&nmi_user_regs);struct pt_regs *user_regs =task_pt_regs (current); +if (!in_nmi ()) { + regs_user->regs = user_regs; + regs_user->abi =perf_reg_abi (current); +return ; + } +/* * If we're in an NMI that interrupted task_pt_regs setup, then * we can't sample user regs at all. This check isn't really
include/linux/perf_event.h +0 -6
@@ -1022,13 +1022,7 @@ struct perf_sample_data {struct perf_callchain_entry *callchain;u64 aux_size; -/* -* regs_user may point to task_pt_regs or to regs_user_copy, depending -* on arch details. -*/ struct perf_regs regs_user; -struct pt_regs regs_user_copy; -struct perf_regs regs_intr;u64 stack_user_size;
include/linux/perf_regs.h +2 -4
@@ -20,8 +20,7 @@ u64 perf_reg_value(struct pt_regs *regs,int perf_reg_validate (u64 mask);u64 perf_reg_abi (struct task_struct *task);void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy); +struct pt_regs *regs);#else #define PERF_REG_EXTENDED_MASK 0
@@ -42,8 +41,7 @@ static inline u64 perf_reg_abi(struct ta }static inline void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs); { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
kernel/events/core.c +3 -5
@@ -6374,14 +6374,13 @@ perf_output_sample_regs(struct perf_outp }static void perf_sample_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) {if (user_mode (regs)) { regs_user->abi =perf_reg_abi (current); regs_user->regs = regs; }else if (!(current->flags &PF_KTHREAD )) { -perf_get_regs_user (regs_user, regs, regs_user_copy); +perf_get_regs_user (regs_user, regs); }else { regs_user->abi =PERF_SAMPLE_REGS_ABI_NONE ; regs_user->regs =NULL ;
@@ -7083,8 +7082,7 @@ void perf_prepare_sample(struct perf_eve }if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) -perf_sample_regs_user (&data->regs_user, regs, - &data->regs_user_copy); +perf_sample_regs_user (&data->regs_user, regs);if (sample_type &PERF_SAMPLE_REGS_USER ) {/* regs dump ABI info */
[tip: perf/urgent] perf/arch: Remove perf_sample_data::regs_user_copy
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: 76a4efa80900fc40e0fdf243b42aec9fb8c35d24 Gitweb: https://git.kernel.org/tip/76a4efa80900fc40e0fdf243b42aec9fb8c35d24 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 12:14:21 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00
perf/arch: Remove perf_sample_data::regs_user_copy
struct perf_sample_data lives on-stack, we should be careful about it's size. Furthermore, the pt_regs copy in there is only because x86_64 is a trainwreck, solve it differently.
--- arch/arm/kernel/perf_regs.c | 3 +-- arch/arm64/kernel/perf_regs.c | 3 +-- arch/csky/kernel/perf_regs.c | 3 +-- arch/powerpc/perf/perf_regs.c | 3 +-- arch/riscv/kernel/perf_regs.c | 3 +-- arch/s390/kernel/perf_regs.c | 3 +-- arch/x86/kernel/perf_regs.c | 15 +++++++++++---- include/linux/perf_event.h | 6 ------ include/linux/perf_regs.h | 6 ++---- kernel/events/core.c | 8 +++----- 10 files changed, 22 insertions(+), 31 deletions(-)
arch/arm/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/arm64/kernel/perf_regs.c +1 -2
@@ -73,8 +73,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/csky/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/powerpc/perf/perf_regs.c +1 -2
@@ -144,8 +144,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi = (regs_user->regs) ? perf_reg_abi(current) :
arch/riscv/kernel/perf_regs.c +1 -2
@@ -36,8 +36,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
arch/s390/kernel/perf_regs.c +1 -2
@@ -53,8 +53,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) {/* * Use the regs from the first interruption and let
arch/x86/kernel/perf_regs.c +11 -4
@@ -101,8 +101,7 @@ u64 perf_reg_abi(struct task_struct *task) }void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
@@ -129,12 +128,20 @@ u64 perf_reg_abi(struct task_struct *task)return PERF_SAMPLE_REGS_ABI_64 ; } +static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs); +void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { +struct pt_regs *regs_user_copy =this_cpu_ptr (&nmi_user_regs);struct pt_regs *user_regs =task_pt_regs (current); +if (!in_nmi ()) { + regs_user->regs = user_regs; + regs_user->abi =perf_reg_abi (current); +return ; + } +/* * If we're in an NMI that interrupted task_pt_regs setup, then * we can't sample user regs at all. This check isn't really
include/linux/perf_event.h +0 -6
@@ -1022,13 +1022,7 @@ struct perf_sample_data {struct perf_callchain_entry *callchain;u64 aux_size; -/* -* regs_user may point to task_pt_regs or to regs_user_copy, depending -* on arch details. -*/ struct perf_regs regs_user; -struct pt_regs regs_user_copy; -struct perf_regs regs_intr;u64 stack_user_size;
include/linux/perf_regs.h +2 -4
@@ -20,8 +20,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx);int perf_reg_validate (u64 mask);u64 perf_reg_abi (struct task_struct *task);void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy); +struct pt_regs *regs);#else #define PERF_REG_EXTENDED_MASK 0
@@ -42,8 +41,7 @@ static inline u64 perf_reg_abi(struct task_struct *task) }static inline void perf_get_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) { regs_user->regs =task_pt_regs (current); regs_user->abi =perf_reg_abi (current);
kernel/events/core.c +3 -5
@@ -6374,14 +6374,13 @@ perf_output_sample_regs(struct perf_output_handle *handle, }static void perf_sample_regs_user (struct perf_regs *regs_user, -struct pt_regs *regs, -struct pt_regs *regs_user_copy) +struct pt_regs *regs) {if (user_mode (regs)) { regs_user->abi =perf_reg_abi (current); regs_user->regs = regs; }else if (!(current->flags &PF_KTHREAD )) { -perf_get_regs_user (regs_user, regs, regs_user_copy); +perf_get_regs_user (regs_user, regs); }else { regs_user->abi =PERF_SAMPLE_REGS_ABI_NONE ; regs_user->regs =NULL ;
@@ -7083,8 +7082,7 @@ void perf_prepare_sample(struct perf_event_header *header, }if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) -perf_sample_regs_user (&data->regs_user, regs, - &data->regs_user_copy); +perf_sample_regs_user (&data->regs_user, regs);if (sample_type &PERF_SAMPLE_REGS_USER ) {/* regs dump ABI info */
Peter Zijlstra
maintainerPATCH 6/6 perf/x86: Make dummy_iregs static
In reply to Peter ZijlstraHaving pt_regs on-stack is unfortunate, it's 168 bytes. Since it isn't actually used, make it a static variable. This both gets if off the stack and ensures it gets 0 initialized, just in case someone does look at it.
--- arch/x86/events/intel/ds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/events/intel/ds.c +1 -1
@@ -1735,7 +1735,7 @@ static void __intel_pmu_pebs_event(strucstruct x86_perf_regs perf_regs;struct pt_regs *regs = &perf_regs.regs;void *at =get_next_pebs_record_by_bit (base, top, bit); -struct pt_regs dummy_iregs; +static struct pt_regs dummy_iregs;if (hwc->flags &PERF_X86_EVENT_AUTO_RELOAD ) {/*
[tip: perf/urgent] perf/x86: Make dummy_iregs static
The following commit has been merged into the perf/urgent branch of tip:
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: e506d1dac0edb2df82f2aa0582e814f9cd9aa07d Gitweb: https://git.kernel.org/tip/e506d1dac0edb2df82f2aa0582e814f9cd9aa07d Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 30 Oct 2020 12:15:06 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 09 Nov 2020 18:12:35 +01:00
perf/x86: Make dummy_iregs static
Having pt_regs on-stack is unfortunate, it's 168 bytes. Since it isn't actually used, make it a static variable. This both gets if off the stack and ensures it gets 0 initialized, just in case someone does look at it.
--- arch/x86/events/intel/ds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/events/intel/ds.c +1 -1
@@ -1738,7 +1738,7 @@ __intel_pmu_pebs_event(struct perf_event *event,struct x86_perf_regs perf_regs;struct pt_regs *regs = &perf_regs.regs;void *at =get_next_pebs_record_by_bit (base, top, bit); -struct pt_regs dummy_iregs; +static struct pt_regs dummy_iregs;if (hwc->flags &PERF_X86_EVENT_AUTO_RELOAD ) {/*