Skip to contentSkip to conversation

perf: Reduce stack usage (and misc bits)

PATCH Applied Peter Zijlstra 6 participants 23 messages Oct 30, 2020 – Nov 10, 2020

Subsystems (9)
Arm (32-bit) x86 PowerPC perf events +5 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover perf: Reduce stack usage (and misc bits)

Hi,

Steve reported an NMI stack overflow when running perf and function tracing together. Thomas found that we had 4 copies of struct perf_sample_data on-stack.

These here patches reduce that to 2 copies and half the size of it.

So just for perf_sample_data we go from 4*384=1536 to 2*192=384 bytes of stack. Also remove one struct pt_regs instance from __intel_pmu_pebs_event(); it has another instance in struct x86_perf_regs which I haven't yet managed to offload.

Perf seems to still work... :-)

author maintainer
#1 lore raw

PATCH 1/6 perf: Reduce stack usage of perf_output_begin()

__perf_output_begin() has an on-stack struct perf_sample_data in the unlikely case it needs to generate a LOST record. However, every call to perf_output_begin() must already have a perf_sample_data on-stack.

---
 arch/powerpc/perf/imc-pmu.c     |    2 +-
 arch/s390/kernel/perf_cpum_sf.c |    2 +-
 arch/x86/events/intel/ds.c      |    4 ++--
 include/linux/perf_event.h      |    7 +++++--
 kernel/events/core.c            |   32 +++++++++++++++++---------------
 kernel/events/ring_buffer.c     |   20 +++++++++++---------
 6 files changed, 37 insertions(+), 30 deletions(-)
arch/powerpc/perf/imc-pmu.c +1 -1
@@ -1336,7 +1336,7 @@ static void dump_trace_imc_data(struct p
 			/* If this is a valid record, create the sample */
 			struct perf_output_handle handle;
 
-			if (perf_output_begin(&handle, event, header.size))
+			if (perf_output_begin(&handle, &data, event, header.size))
 				return;
 
 			perf_output_sample(&handle, &header, &data, event);
arch/s390/kernel/perf_cpum_sf.c +1 -1
@@ -672,7 +672,7 @@ static void cpumsf_output_event_pid(stru
 	rcu_read_lock();
 
 	perf_prepare_sample(&header, data, event, regs);
-	if (perf_output_begin(&handle, event, header.size))
+	if (perf_output_begin(&handle, data, event, header.size))
 		goto out;
 
 	/* Update the process ID (see also kernel/events/core.c) */
arch/x86/events/intel/ds.c +2 -2
@@ -642,8 +642,8 @@ int intel_pmu_drain_bts_buffer(void)
 	rcu_read_lock();
 	perf_prepare_sample(&header, &data, event, &regs);
 
-	if (perf_output_begin(&handle, event, header.size *
-			      (top - base - skip)))
+	if (perf_output_begin(&handle, &data, event,
+			      header.size * (top - base - skip)))
 		goto unlock;
 
 	for (at = base; at < top; at++) {
include/linux/perf_event.h +5 -2
@@ -1400,11 +1400,14 @@ perf_event_addr_filters(struct perf_even
 extern void perf_event_addr_filters_sync(struct perf_event *event);
 
 extern int perf_output_begin(struct perf_output_handle *handle,
+			     struct perf_sample_data *data,
 			     struct perf_event *event, unsigned int size);
 extern int perf_output_begin_forward(struct perf_output_handle *handle,
-				    struct perf_event *event,
-				    unsigned int size);
+				     struct perf_sample_data *data,
+				     struct perf_event *event,
+				     unsigned int size);
 extern int perf_output_begin_backward(struct perf_output_handle *handle,
+				      struct perf_sample_data *data,
 				      struct perf_event *event,
 				      unsigned int size);
 
kernel/events/core.c +17 -15
@@ -7186,6 +7186,7 @@ __perf_event_output(struct perf_event *e
 		    struct perf_sample_data *data,
 		    struct pt_regs *regs,
 		    int (*output_begin)(struct perf_output_handle *,
+					struct perf_sample_data *,
 					struct perf_event *,
 					unsigned int))
 {
@@ -7198,7 +7199,7 @@ __perf_event_output(struct perf_event *e
 
 	perf_prepare_sample(&header, data, event, regs);
 
-	err = output_begin(&handle, event, header.size);
+	err = output_begin(&handle, data, event, header.size);
 	if (err)
 		goto exit;
 
@@ -7264,7 +7265,7 @@ perf_event_read_event(struct perf_event
 	int ret;
 
 	perf_event_header__init_id(&read_event.header, &sample, event);
-	ret = perf_output_begin(&handle, event, read_event.header.size);
+	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
 	if (ret)
 		return;
 
@@ -7533,7 +7534,7 @@ static void perf_event_task_output(struc
 
 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				task_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7636,7 +7637,7 @@ static void perf_event_comm_output(struc
 		return;
 
 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				comm_event->event_id.header.size);
 
 	if (ret)
@@ -7736,7 +7737,7 @@ static void perf_event_namespaces_output
 
 	perf_event_header__init_id(&namespaces_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				namespaces_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7863,7 +7864,7 @@ static void perf_event_cgroup_output(str
 
 	perf_event_header__init_id(&cgroup_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				cgroup_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7989,7 +7990,7 @@ static void perf_event_mmap_output(struc
 	}
 
 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				mmap_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -8299,7 +8300,7 @@ void perf_event_aux_event(struct perf_ev
 	int ret;
 
 	perf_event_header__init_id(&rec.header, &sample, event);
-	ret = perf_output_begin(&handle, event, rec.header.size);
+	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
 
 	if (ret)
 		return;
@@ -8333,7 +8334,7 @@ void perf_log_lost_samples(struct perf_e
 
 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				lost_samples_event.header.size);
 	if (ret)
 		return;
@@ -8388,7 +8389,7 @@ static void perf_event_switch_output(str
 
 	perf_event_header__init_id(&se->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event, se->event_id.header.size);
+	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
 	if (ret)
 		return;
 
@@ -8463,7 +8464,7 @@ static void perf_log_throttle(struct per
 
 	perf_event_header__init_id(&throttle_event.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				throttle_event.header.size);
 	if (ret)
 		return;
@@ -8506,7 +8507,7 @@ static void perf_event_ksymbol_output(st
 
 	perf_event_header__init_id(&ksymbol_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				ksymbol_event->event_id.header.size);
 	if (ret)
 		return;
@@ -8596,7 +8597,7 @@ static void perf_event_bpf_output(struct
 
 	perf_event_header__init_id(&bpf_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, data, event,
 				bpf_event->event_id.header.size);
 	if (ret)
 		return;
@@ -8705,7 +8706,8 @@ static void perf_event_text_poke_output(
 
 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event, text_poke_event->event_id.header.size);
+	ret = perf_output_begin(&handle, &sample, event,
+				text_poke_event->event_id.header.size);
 	if (ret)
 		return;
 
@@ -8786,7 +8788,7 @@ static void perf_log_itrace_start(struct
 	rec.tid	= perf_event_tid(event, current);
 
 	perf_event_header__init_id(&rec.header, &sample, event);
-	ret = perf_output_begin(&handle, event, rec.header.size);
+	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
 
 	if (ret)
 		return;
kernel/events/ring_buffer.c +11 -9
@@ -147,6 +147,7 @@ ring_buffer_has_space(unsigned long head
 
 static __always_inline int
 __perf_output_begin(struct perf_output_handle *handle,
+		    struct perf_sample_data *data,
 		    struct perf_event *event, unsigned int size,
 		    bool backward)
 {
@@ -237,18 +238,16 @@ __perf_output_begin(struct perf_output_h
 	handle->size = (1UL << page_shift) - offset;
 
 	if (unlikely(have_lost)) {
-		struct perf_sample_data sample_data;
-
 		lost_event.header.size = sizeof(lost_event);
 		lost_event.header.type = PERF_RECORD_LOST;
 		lost_event.header.misc = 0;
 		lost_event.id          = event->id;
 		lost_event.lost        = local_xchg(&rb->lost, 0);
 
-		perf_event_header__init_id(&lost_event.header,
-					   &sample_data, event);
+		/* XXX mostly redundant; @data is already fully initializes */
+		perf_event_header__init_id(&lost_event.header, data, event);
 		perf_output_put(handle, lost_event);
-		perf_event__output_id_sample(event, handle, &sample_data);
+		perf_event__output_id_sample(event, handle, data);
 	}
 
 	return 0;
@@ -263,22 +262,25 @@ __perf_output_begin(struct perf_output_h
 }
 
 int perf_output_begin_forward(struct perf_output_handle *handle,
-			     struct perf_event *event, unsigned int size)
+			      struct perf_sample_data *data,
+			      struct perf_event *event, unsigned int size)
 {
-	return __perf_output_begin(handle, event, size, false);
+	return __perf_output_begin(handle, data, event, size, false);
 }
 
 int perf_output_begin_backward(struct perf_output_handle *handle,
+			       struct perf_sample_data *data,
 			       struct perf_event *event, unsigned int size)
 {
-	return __perf_output_begin(handle, event, size, true);
+	return __perf_output_begin(handle, data, event, size, true);
 }
 
 int perf_output_begin(struct perf_output_handle *handle,
+		      struct perf_sample_data *data,
 		      struct perf_event *event, unsigned int size)
 {
 
-	return __perf_output_begin(handle, event, size,
+	return __perf_output_begin(handle, data, event, size,
 				   unlikely(is_write_backward(event)));
 }
 
merge bot Applied
#2 lore raw

[tip: perf/urgent] perf: Reduce stack usage of perf_output_begin()

The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     267fb27352b6fc9fdbad753127a239f75618ecbc
Gitweb:        https://git.kernel.org/tip/267fb27352b6fc9fdbad753127a239f75618ecbc
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 15:50:32 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:33 +01:00

perf: Reduce stack usage of perf_output_begin()

__perf_output_begin() has an on-stack struct perf_sample_data in the unlikely case it needs to generate a LOST record. However, every call to perf_output_begin() must already have a perf_sample_data on-stack.

---
 arch/powerpc/perf/imc-pmu.c     |  2 +-
 arch/s390/kernel/perf_cpum_sf.c |  2 +-
 arch/x86/events/intel/ds.c      |  4 ++--
 include/linux/perf_event.h      |  7 +++++--
 kernel/events/core.c            | 32 +++++++++++++++++---------------
 kernel/events/ring_buffer.c     | 20 +++++++++++---------
 6 files changed, 37 insertions(+), 30 deletions(-)
arch/powerpc/perf/imc-pmu.c +1 -1
index 9ed4fcc..7b25548 100644
@@ -1336,7 +1336,7 @@ static void dump_trace_imc_data(struct perf_event *event)
 			/* If this is a valid record, create the sample */
 			struct perf_output_handle handle;
 
-			if (perf_output_begin(&handle, event, header.size))
+			if (perf_output_begin(&handle, &data, event, header.size))
 				return;
 
 			perf_output_sample(&handle, &header, &data, event);
arch/s390/kernel/perf_cpum_sf.c +1 -1
index 4f9e462..00255ae 100644
@@ -672,7 +672,7 @@ static void cpumsf_output_event_pid(struct perf_event *event,
 	rcu_read_lock();
 
 	perf_prepare_sample(&header, data, event, regs);
-	if (perf_output_begin(&handle, event, header.size))
+	if (perf_output_begin(&handle, data, event, header.size))
 		goto out;
 
 	/* Update the process ID (see also kernel/events/core.c) */
arch/x86/events/intel/ds.c +2 -2
index 404315d..cd2ae14 100644
@@ -642,8 +642,8 @@ int intel_pmu_drain_bts_buffer(void)
 	rcu_read_lock();
 	perf_prepare_sample(&header, &data, event, &regs);
 
-	if (perf_output_begin(&handle, event, header.size *
-			      (top - base - skip)))
+	if (perf_output_begin(&handle, &data, event,
+			      header.size * (top - base - skip)))
 		goto unlock;
 
 	for (at = base; at < top; at++) {
include/linux/perf_event.h +5 -2
index 0c19d27..b775ae0 100644
@@ -1400,11 +1400,14 @@ perf_event_addr_filters(struct perf_event *event)
 extern void perf_event_addr_filters_sync(struct perf_event *event);
 
 extern int perf_output_begin(struct perf_output_handle *handle,
+			     struct perf_sample_data *data,
 			     struct perf_event *event, unsigned int size);
 extern int perf_output_begin_forward(struct perf_output_handle *handle,
-				    struct perf_event *event,
-				    unsigned int size);
+				     struct perf_sample_data *data,
+				     struct perf_event *event,
+				     unsigned int size);
 extern int perf_output_begin_backward(struct perf_output_handle *handle,
+				      struct perf_sample_data *data,
 				      struct perf_event *event,
 				      unsigned int size);
 
kernel/events/core.c +17 -15
index 5a29ab0..fc681c7 100644
@@ -7186,6 +7186,7 @@ __perf_event_output(struct perf_event *event,
 		    struct perf_sample_data *data,
 		    struct pt_regs *regs,
 		    int (*output_begin)(struct perf_output_handle *,
+					struct perf_sample_data *,
 					struct perf_event *,
 					unsigned int))
 {
@@ -7198,7 +7199,7 @@ __perf_event_output(struct perf_event *event,
 
 	perf_prepare_sample(&header, data, event, regs);
 
-	err = output_begin(&handle, event, header.size);
+	err = output_begin(&handle, data, event, header.size);
 	if (err)
 		goto exit;
 
@@ -7264,7 +7265,7 @@ perf_event_read_event(struct perf_event *event,
 	int ret;
 
 	perf_event_header__init_id(&read_event.header, &sample, event);
-	ret = perf_output_begin(&handle, event, read_event.header.size);
+	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
 	if (ret)
 		return;
 
@@ -7533,7 +7534,7 @@ static void perf_event_task_output(struct perf_event *event,
 
 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				task_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7636,7 +7637,7 @@ static void perf_event_comm_output(struct perf_event *event,
 		return;
 
 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				comm_event->event_id.header.size);
 
 	if (ret)
@@ -7736,7 +7737,7 @@ static void perf_event_namespaces_output(struct perf_event *event,
 
 	perf_event_header__init_id(&namespaces_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				namespaces_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7863,7 +7864,7 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data)
 
 	perf_event_header__init_id(&cgroup_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				cgroup_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -7989,7 +7990,7 @@ static void perf_event_mmap_output(struct perf_event *event,
 	}
 
 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				mmap_event->event_id.header.size);
 	if (ret)
 		goto out;
@@ -8299,7 +8300,7 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head,
 	int ret;
 
 	perf_event_header__init_id(&rec.header, &sample, event);
-	ret = perf_output_begin(&handle, event, rec.header.size);
+	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
 
 	if (ret)
 		return;
@@ -8333,7 +8334,7 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost)
 
 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				lost_samples_event.header.size);
 	if (ret)
 		return;
@@ -8388,7 +8389,7 @@ static void perf_event_switch_output(struct perf_event *event, void *data)
 
 	perf_event_header__init_id(&se->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event, se->event_id.header.size);
+	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
 	if (ret)
 		return;
 
@@ -8463,7 +8464,7 @@ static void perf_log_throttle(struct perf_event *event, int enable)
 
 	perf_event_header__init_id(&throttle_event.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				throttle_event.header.size);
 	if (ret)
 		return;
@@ -8506,7 +8507,7 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data)
 
 	perf_event_header__init_id(&ksymbol_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, &sample, event,
 				ksymbol_event->event_id.header.size);
 	if (ret)
 		return;
@@ -8596,7 +8597,7 @@ static void perf_event_bpf_output(struct perf_event *event, void *data)
 
 	perf_event_header__init_id(&bpf_event->event_id.header,
 				   &sample, event);
-	ret = perf_output_begin(&handle, event,
+	ret = perf_output_begin(&handle, data, event,
 				bpf_event->event_id.header.size);
 	if (ret)
 		return;
@@ -8705,7 +8706,8 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data)
 
 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
 
-	ret = perf_output_begin(&handle, event, text_poke_event->event_id.header.size);
+	ret = perf_output_begin(&handle, &sample, event,
+				text_poke_event->event_id.header.size);
 	if (ret)
 		return;
 
@@ -8786,7 +8788,7 @@ static void perf_log_itrace_start(struct perf_event *event)
 	rec.tid	= perf_event_tid(event, current);
 
 	perf_event_header__init_id(&rec.header, &sample, event);
-	ret = perf_output_begin(&handle, event, rec.header.size);
+	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
 
 	if (ret)
 		return;
kernel/events/ring_buffer.c +11 -9
index 192b8ab..ef91ae7 100644
@@ -147,6 +147,7 @@ ring_buffer_has_space(unsigned long head, unsigned long tail,
 
 static __always_inline int
 __perf_output_begin(struct perf_output_handle *handle,
+		    struct perf_sample_data *data,
 		    struct perf_event *event, unsigned int size,
 		    bool backward)
 {
@@ -237,18 +238,16 @@ __perf_output_begin(struct perf_output_handle *handle,
 	handle->size = (1UL << page_shift) - offset;
 
 	if (unlikely(have_lost)) {
-		struct perf_sample_data sample_data;
-
 		lost_event.header.size = sizeof(lost_event);
 		lost_event.header.type = PERF_RECORD_LOST;
 		lost_event.header.misc = 0;
 		lost_event.id          = event->id;
 		lost_event.lost        = local_xchg(&rb->lost, 0);
 
-		perf_event_header__init_id(&lost_event.header,
-					   &sample_data, event);
+		/* XXX mostly redundant; @data is already fully initializes */
+		perf_event_header__init_id(&lost_event.header, data, event);
 		perf_output_put(handle, lost_event);
-		perf_event__output_id_sample(event, handle, &sample_data);
+		perf_event__output_id_sample(event, handle, data);
 	}
 
 	return 0;
@@ -263,22 +262,25 @@ out:
 }
 
 int perf_output_begin_forward(struct perf_output_handle *handle,
-			     struct perf_event *event, unsigned int size)
+			      struct perf_sample_data *data,
+			      struct perf_event *event, unsigned int size)
 {
-	return __perf_output_begin(handle, event, size, false);
+	return __perf_output_begin(handle, data, event, size, false);
 }
 
 int perf_output_begin_backward(struct perf_output_handle *handle,
+			       struct perf_sample_data *data,
 			       struct perf_event *event, unsigned int size)
 {
-	return __perf_output_begin(handle, event, size, true);
+	return __perf_output_begin(handle, data, event, size, true);
 }
 
 int perf_output_begin(struct perf_output_handle *handle,
+		      struct perf_sample_data *data,
 		      struct perf_event *event, unsigned int size)
 {
 
-	return __perf_output_begin(handle, event, size,
+	return __perf_output_begin(handle, data, event, size,
 				   unlikely(is_write_backward(event)));
 }
 
author maintainer
#3 lore raw

PATCH 2/6 perf/x86: Reduce stack usage for x86_pmu::drain_pebs()

In reply to Peter Zijlstra

intel_pmu_drain_pebs_*() is typically called from handle_pmi_common(), both have an on-stack struct perf_sample_data, which is *big*. Rewire things so that drain_pebs() can use the one handle_pmi_common() has.

---
 arch/x86/events/intel/core.c |    2 -
 arch/x86/events/intel/ds.c   |   47 ++++++++++++++++++++++---------------------
 arch/x86/events/perf_event.h |    2 -
 3 files changed, 27 insertions(+), 24 deletions(-)
arch/x86/events/intel/core.c +1 -1
@@ -2630,7 +2630,7 @@ static int handle_pmi_common(struct pt_r
 		u64 pebs_enabled = cpuc->pebs_enabled;
 
 		handled++;
-		x86_pmu.drain_pebs(regs);
+		x86_pmu.drain_pebs(regs, &data);
 		status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
 
 		/*
arch/x86/events/intel/ds.c +25 -22

118 lines not on this page · raw message

arch/x86/events/perf_event.h +1 -1
@@ -727,7 +727,7 @@ struct x86_pmu {
 	int		pebs_record_size;
 	int		pebs_buffer_size;
 	int		max_pebs_events;
-	void		(*drain_pebs)(struct pt_regs *regs);
+	void		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	unsigned long	large_pebs_flags;
merge bot Applied
#4 lore raw

[tip: perf/urgent] perf/x86: Reduce stack usage for x86_pmu::drain_pebs()

The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     9dfa9a5c9bae3417b87824e7ac73b00c10b6a874
Gitweb:        https://git.kernel.org/tip/9dfa9a5c9bae3417b87824e7ac73b00c10b6a874
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 14:58:48 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:33 +01:00

perf/x86: Reduce stack usage for x86_pmu::drain_pebs()

intel_pmu_drain_pebs_*() is typically called from handle_pmi_common(), both have an on-stack struct perf_sample_data, which is *big*. Rewire things so that drain_pebs() can use the one handle_pmi_common() has.

---
 arch/x86/events/intel/core.c |  2 +-
 arch/x86/events/intel/ds.c   | 47 ++++++++++++++++++-----------------
 arch/x86/events/perf_event.h |  2 +-
 3 files changed, 27 insertions(+), 24 deletions(-)
arch/x86/events/intel/core.c +1 -1
index f1926e9..c37387c 100644
@@ -2630,7 +2630,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 		u64 pebs_enabled = cpuc->pebs_enabled;
 
 		handled++;
-		x86_pmu.drain_pebs(regs);
+		x86_pmu.drain_pebs(regs, &data);
 		status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
 
 		/*
arch/x86/events/intel/ds.c +25 -22

118 lines not on this page · raw message

arch/x86/events/perf_event.h +1 -1
index ee2b9b9..1d1fe46 100644
@@ -727,7 +727,7 @@ struct x86_pmu {
 	int		pebs_record_size;
 	int		pebs_buffer_size;
 	int		max_pebs_events;
-	void		(*drain_pebs)(struct pt_regs *regs);
+	void		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	unsigned long	large_pebs_flags;
author maintainer
#5 lore raw

PATCH 3/6 perf: Fix get_recursion_context()

In reply to Peter Zijlstra

One should use in_server_softirq() to detector SoftIRQ context.

Fixes 96f6d4444302 ("perf_counter: avoid recursion")
---
 kernel/events/internal.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/events/internal.h +1 -1
@@ -211,7 +211,7 @@ static inline int get_recursion_context(
 		rctx = 3;
 	else if (in_irq())
 		rctx = 2;
-	else if (in_softirq())
+	else if (in_serving_softirq())
 		rctx = 1;
 	else
 		rctx = 0;
merge bot Applied
#6 lore raw

[tip: perf/urgent] perf: Fix get_recursion_context()

The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     ce0f17fc93f63ee91428af10b7b2ddef38cd19e5
Gitweb:        https://git.kernel.org/tip/ce0f17fc93f63ee91428af10b7b2ddef38cd19e5
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 12:49:45 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00

perf: Fix get_recursion_context()

One should use in_serving_softirq() to detect SoftIRQ context.

Fixes 96f6d4444302 ("perf_counter: avoid recursion")
---
 kernel/events/internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/events/internal.h +1 -1
index fcbf561..402054e 100644
@@ -211,7 +211,7 @@ static inline int get_recursion_context(int *recursion)
 		rctx = 3;
 	else if (in_irq())
 		rctx = 2;
-	else if (in_softirq())
+	else if (in_serving_softirq())
 		rctx = 1;
 	else
 		rctx = 0;
author maintainer
#7 lore raw

PATCH 4/6 perf: Optimize get_recursion_context()

In reply to Peter Zijlstra

"Look ma, no branches!"

Cc Jesper Dangaard Brouer <[email protected]>
Cc Steven Rostedt <[email protected]>
---
 kernel/events/internal.h |   17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/events/internal.h +8 -9
@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, a
 
 static inline int get_recursion_context(int *recursion)
 {
-	int rctx;
+	unsigned int pc = preempt_count();
+	unsigned int rctx = 0;
 
-	if (unlikely(in_nmi()))
-		rctx = 3;
-	else if (in_irq())
-		rctx = 2;
-	else if (in_serving_softirq())
-		rctx = 1;
-	else
-		rctx = 0;
+	if (pc & (NMI_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
+		rctx++;
 
 	if (recursion[rctx])
 		return -1;
Acked-by on v1 4/6
#8 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 6 lines · On Fri, 30 Oct 2020 16:13:49 +0100 Peter Zijlstra <[email protected]> wrote:

"Look ma, no branches!"

Cc Jesper Dangaard Brouer <[email protected]>
Cc Steven Rostedt <[email protected]>

---

Cool trick! :-)

Quoted 32 lines
 kernel/events/internal.h |   17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/events/internal.h +8 -9
@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, a
 
 static inline int get_recursion_context(int *recursion)
 {
-	int rctx;
+	unsigned int pc = preempt_count();
+	unsigned int rctx = 0;
 
-	if (unlikely(in_nmi()))
-		rctx = 3;
-	else if (in_irq())
-		rctx = 2;
-	else if (in_serving_softirq())
-		rctx = 1;
-	else
-		rctx = 0;
+	if (pc & (NMI_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
+		rctx++;
 
 	if (recursion[rctx])
 		return -1;
Signature · 4 lines
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer
#9 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 42 lines · On Fri, 30 Oct 2020 18:11:38 +0100 Jesper Dangaard Brouer <[email protected]>…
Quoted 6 lines · On Fri, 30 Oct 2020 16:13:49 +0100 Peter Zijlstra <[email protected]> wrote:

"Look ma, no branches!"

Cc Jesper Dangaard Brouer <[email protected]>
Cc Steven Rostedt <[email protected]>

---

Cool trick! :-)

Quoted 28 lines
 kernel/events/internal.h |   17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/events/internal.h +8 -9
@@ -205,16 +205,15 @@ DEFINE_OUTPUT_COPY(__output_copy_user, a
 
 static inline int get_recursion_context(int *recursion)
 {
-	int rctx;
+	unsigned int pc = preempt_count();
+	unsigned int rctx = 0;
 
-	if (unlikely(in_nmi()))
-		rctx = 3;
-	else if (in_irq())
-		rctx = 2;
-	else if (in_serving_softirq())
-		rctx = 1;
-	else
-		rctx = 0;
+	if (pc & (NMI_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK))
+		rctx++;
+	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
+		rctx++;
 
Quoted 4 lines

kernel/events/internal.h DEFINE_OUTPUT_COPY(__output_copy_user, a · line 219

Quoted 4 lines
kernel/events/internal.h +0 -0
 	if (recursion[rctx])
 		return -1;

As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get it quickly, and not keep re-implementing it.

/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline int irq_context()

{
	unsigned int pc = preempt_count();
	int rctx = 0;

	if (pc & (NMI_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
		rctx++;

	return rctx;
}

-- Steve

#10 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 2 lines · Steven Rostedt

As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get

Not in interrupt.h please. We should create kernel/include/ for stuff which really should only be available in the core kernel code.

Thanks,

tglx

#11 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 7 lines · On Fri, 30 Oct 2020 23:14:18 +0100 Thomas Gleixner <[email protected]> wrote:
Quoted 2 lines · Steven Rostedt

As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get

Not in interrupt.h please. We should create kernel/include/ for stuff which really should only be available in the core kernel code.

The recursion protection is needed for anything that registers a callback to ftrace. I have patches that already basically do the same thing (although, with branches) that I'm going to place in include/linux/trace_recursion.h, so that there are helper functions that ftrace callbacks can use, instead of having to implement their own recursion protection. After fixing two bugs in the recursion protection code, it's something that should be reused, instead of everyone making similar mistakes.

Note, I thought that in_nmi() and friends was in interrupt.h, but is really in preempt.h. All the values used in Peter's code is also defined in preempt.h, so why not have something like that there?

I take back adding it to interrupt.h but have it in preempt.h, as it's not defining anything new there.

-- Steve

author maintainer
#12 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 6 lines · Steven Rostedt

Note, I thought that in_nmi() and friends was in interrupt.h, but is really in preempt.h. All the values used in Peter's code is also defined in preempt.h, so why not have something like that there?

I take back adding it to interrupt.h but have it in preempt.h, as it's not defining anything new there.

Yeah, preempt.h is the right place for it.

author maintainer
#13 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

In reply to Steven Rostedt
Quoted 3 lines · Steven Rostedt

As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get it quickly, and not keep re-implementing it.

Works for me, however:

Quoted 11 lines

/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline int irq_context()

{
	unsigned int pc = preempt_count();
	int rctx = 0;

unsigned

Quoted 10 lines
	if (pc & (NMI_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
		rctx++;

	return rctx;
}

otherwise you'll get an extra instruction to sign extend it, which is daft (yes, i've been staring at GCC output far too much).

Also, gcc-9 does worse (like 1 byte iirc) with:

	rctx += !!(pc & (NMI_MASK));
	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

but gcc-10 doesn't seem to care.

#14 lore raw

RE: [PATCH 4/6] perf: Optimize get_recursion_context()

From: Peter Zijlstra

Quoted 37 lines

Sent: 30 October 2020 23:02

Quoted 3 lines · Steven Rostedt

As this is something that ftrace recursion also does, perhaps we should move this into interrupt.h so that anyone that needs a counter can get it quickly, and not keep re-implementing it.

Works for me, however:

Quoted 11 lines

/*
* Quickly find what context you are in.
* 0 - normal
* 1 - softirq
* 2 - hard interrupt
* 3 - NMI
*/
static inline int irq_context()

{
	unsigned int pc = preempt_count();
	int rctx = 0;

unsigned

Quoted 10 lines
	if (pc & (NMI_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK))
		rctx++;
	if (pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET))
		rctx++;

	return rctx;
}

otherwise you'll get an extra instruction to sign extend it, which is daft (yes, i've been staring at GCC output far too much).

Quoted 7 lines

Also, gcc-9 does worse (like 1 byte iirc) with:

	rctx += !!(pc & (NMI_MASK));
	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

but gcc-10 doesn't seem to care.

You've made be look at some gcc output (it's raining).

The gcc 7.5.0 I have handy probably generates the best code for:

unsigned char q_2(unsigned int pc)

{
        unsigned char rctx = 0;

        rctx += !!(pc & (NMI_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return rctx;
}

0000000000000000 <q_2>:

   0:   f7 c7 00 00 f0 00       test   $0xf00000,%edi     # clock 0
   6:   0f 95 c0                setne  %al                # clock 1
   9:   f7 c7 00 00 ff 00       test   $0xff0000,%edi     # clock 0
   f:   0f 95 c2                setne  %dl                # clock 1
  12:   01 c2                   add    %eax,%edx          # clock 2
  14:   81 e7 00 01 ff 00       and    $0xff0100,%edi
  1a:   0f 95 c0                setne  %al
  1d:   01 d0                   add    %edx,%eax          # clock 3
  1f:   c3                      retq

I doubt that is beatable.

I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

#15 lore raw

RE: [PATCH 4/6] perf: Optimize get_recursion_context()

From: David Laight

Quoted 2 lines

Sent: 31 October 2020 12:12

...

Quoted 24 lines

The gcc 7.5.0 I have handy probably generates the best code for:

unsigned char q_2(unsigned int pc)

{
        unsigned char rctx = 0;

        rctx += !!(pc & (NMI_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return rctx;
}

0000000000000000 <q_2>:

   0:   f7 c7 00 00 f0 00       test   $0xf00000,%edi     # clock 0
   6:   0f 95 c0                setne  %al                # clock 1
   9:   f7 c7 00 00 ff 00       test   $0xff0000,%edi     # clock 0
   f:   0f 95 c2                setne  %dl                # clock 1
  12:   01 c2                   add    %eax,%edx          # clock 2
  14:   81 e7 00 01 ff 00       and    $0xff0100,%edi
  1a:   0f 95 c0                setne  %al
  1d:   01 d0                   add    %edx,%eax          # clock 3
  1f:   c3                      retq
Quoted 1 line

I doubt that is beatable.

I lied, you should be able to get:

	test   $0xff0000,%edi     # clock 0
	setne  %al                # clock 1
	test   $0xff0100,%edi     # clock 0
	setne  %dl                # clock 1
	add    $fffff000,%edi
	adc    %dl, %al           # clock 2

But I suspect getting it from the compiler might be hard!

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

author maintainer
#16 lore raw

Re: [PATCH 4/6] perf: Optimize get_recursion_context()

In reply to David Laight
Quoted 24 lines · David Laight

The gcc 7.5.0 I have handy probably generates the best code for:

unsigned char q_2(unsigned int pc)

{
        unsigned char rctx = 0;

        rctx += !!(pc & (NMI_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return rctx;
}

0000000000000000 <q_2>:

   0:   f7 c7 00 00 f0 00       test   $0xf00000,%edi     # clock 0
   6:   0f 95 c0                setne  %al                # clock 1
   9:   f7 c7 00 00 ff 00       test   $0xff0000,%edi     # clock 0
   f:   0f 95 c2                setne  %dl                # clock 1
  12:   01 c2                   add    %eax,%edx          # clock 2
  14:   81 e7 00 01 ff 00       and    $0xff0100,%edi
  1a:   0f 95 c0                setne  %al
  1d:   01 d0                   add    %edx,%eax          # clock 3
  1f:   c3                      retq
Quoted 6 lines

I doubt that is beatable.

I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.

https://godbolt.org/z/EfnG8E

Recent GCC just doesn't want to do that. Still, using u8 makes sense, so I've kept that.

#17 lore raw

RE: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 44 lines

-----Original Message-----
From: Peter Zijlstra <[email protected]>
Sent: 09 November 2020 12:13
To: David Laight <[email protected]>
Cc: Steven Rostedt <[email protected]>; Jesper Dangaard Brouer <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH 4/6] perf: Optimize get_recursion_context()

Quoted 24 lines · David Laight

The gcc 7.5.0 I have handy probably generates the best code for:

unsigned char q_2(unsigned int pc)

{
        unsigned char rctx = 0;

        rctx += !!(pc & (NMI_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return rctx;
}

0000000000000000 <q_2>:

   0:   f7 c7 00 00 f0 00       test   $0xf00000,%edi     # clock 0
   6:   0f 95 c0                setne  %al                # clock 1
   9:   f7 c7 00 00 ff 00       test   $0xff0000,%edi     # clock 0
   f:   0f 95 c2                setne  %dl                # clock 1
  12:   01 c2                   add    %eax,%edx          # clock 2
  14:   81 e7 00 01 ff 00       and    $0xff0100,%edi
  1a:   0f 95 c0                setne  %al
  1d:   01 d0                   add    %edx,%eax          # clock 3
  1f:   c3                      retq
Quoted 6 lines

I doubt that is beatable.

I've annotated the register dependency chain. Likely to be 3 (or maybe 4) clocks.
The other versions are a lot worse (7 or 8) without allowing for 'sbb' taking 2 clocks on a lot of Intel cpus.

Quoted 4 lines

https://godbolt.org/z/EfnG8E

Recent GCC just doesn't want to do that. Still, using u8 makes sense, so I've kept that.

u8 helps x86 because its 'setne' only affects the low 8 bits. I guess that seemed a good idea when it was added (386). It doesn't seem to make the other architectures much worse.

gcc 10.x can be persuaded to generate the above code.

https://godbolt.org/z/6GoT94

It sometimes seems to me that every new version of gcc is larger, slower and generates worse code than the previous one.

David

-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)

merge bot Applied
#18 lore raw

[tip: perf/urgent] perf: Optimize get_recursion_context()

In reply to Peter Zijlstra
The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     09da9c81253dd8e43e0d2d7cea02de6f9f19499d
Gitweb:        https://git.kernel.org/tip/09da9c81253dd8e43e0d2d7cea02de6f9f19499d
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 13:43:16 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00

perf: Optimize get_recursion_context()

"Look ma, no branches!"

---
 kernel/events/internal.h | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)
kernel/events/internal.h +6 -10
index 402054e..228801e 100644
@@ -205,16 +205,12 @@ DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
 
 static inline int get_recursion_context(int *recursion)
 {
-	int rctx;
-
-	if (unlikely(in_nmi()))
-		rctx = 3;
-	else if (in_irq())
-		rctx = 2;
-	else if (in_serving_softirq())
-		rctx = 1;
-	else
-		rctx = 0;
+	unsigned int pc = preempt_count();
+	unsigned char rctx = 0;
+
+	rctx += !!(pc & (NMI_MASK));
+	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
+	rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
 
 	if (recursion[rctx])
 		return -1;
author maintainer
#19 lore raw

PATCH 5/6 perf/arch: Remove perf_sample_data::regs_user_copy

In reply to Peter Zijlstra

struct perf_sample_data lives on-stack, we should be careful about it's size. Furthermore, the pt_regs copy in there is only because x86_64 is a trainwreck, solve it differently.

---
 arch/arm/kernel/perf_regs.c   |    3 +--
 arch/arm64/kernel/perf_regs.c |    3 +--
 arch/csky/kernel/perf_regs.c  |    3 +--
 arch/powerpc/perf/perf_regs.c |    3 +--
 arch/riscv/kernel/perf_regs.c |    3 +--
 arch/s390/kernel/perf_regs.c  |    3 +--
 arch/x86/kernel/perf_regs.c   |   15 +++++++++++----
 include/linux/perf_event.h    |    6 ------
 include/linux/perf_regs.h     |    6 ++----
 kernel/events/core.c          |    8 +++-----
 10 files changed, 22 insertions(+), 31 deletions(-)
arch/arm/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/arm64/kernel/perf_regs.c +1 -2
@@ -73,8 +73,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/csky/kernel/perf_regs.c +1 -2
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/powerpc/perf/perf_regs.c +1 -2
@@ -144,8 +144,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = (regs_user->regs) ? perf_reg_abi(current) :
arch/riscv/kernel/perf_regs.c +1 -2
@@ -36,8 +36,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/s390/kernel/perf_regs.c +1 -2
@@ -53,8 +53,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	/*
 	 * Use the regs from the first interruption and let
arch/x86/kernel/perf_regs.c +11 -4
@@ -101,8 +101,7 @@ u64 perf_reg_abi(struct task_struct *tas
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
@@ -129,12 +128,20 @@ u64 perf_reg_abi(struct task_struct *tas
 		return PERF_SAMPLE_REGS_ABI_64;
 }
 
+static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs);
+
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
+	struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs);
 	struct pt_regs *user_regs = task_pt_regs(current);
 
+	if (!in_nmi()) {
+		regs_user->regs = user_regs;
+		regs_user->abi = perf_reg_abi(current);
+		return;
+	}
+
 	/*
 	 * If we're in an NMI that interrupted task_pt_regs setup, then
 	 * we can't sample user regs at all.  This check isn't really
include/linux/perf_event.h +0 -6
@@ -1022,13 +1022,7 @@ struct perf_sample_data {
 	struct perf_callchain_entry	*callchain;
 	u64				aux_size;
 
-	/*
-	 * regs_user may point to task_pt_regs or to regs_user_copy, depending
-	 * on arch details.
-	 */
 	struct perf_regs		regs_user;
-	struct pt_regs			regs_user_copy;
-
 	struct perf_regs		regs_intr;
 	u64				stack_user_size;
 
include/linux/perf_regs.h +2 -4
@@ -20,8 +20,7 @@ u64 perf_reg_value(struct pt_regs *regs,
 int perf_reg_validate(u64 mask);
 u64 perf_reg_abi(struct task_struct *task);
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy);
+			struct pt_regs *regs);
 #else
 
 #define PERF_REG_EXTENDED_MASK	0
@@ -42,8 +41,7 @@ static inline u64 perf_reg_abi(struct ta
 }
 
 static inline void perf_get_regs_user(struct perf_regs *regs_user,
-				      struct pt_regs *regs,
-				      struct pt_regs *regs_user_copy)
+				      struct pt_regs *regs);
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
kernel/events/core.c +3 -5
@@ -6374,14 +6374,13 @@ perf_output_sample_regs(struct perf_outp
 }
 
 static void perf_sample_regs_user(struct perf_regs *regs_user,
-				  struct pt_regs *regs,
-				  struct pt_regs *regs_user_copy)
+				  struct pt_regs *regs)
 {
 	if (user_mode(regs)) {
 		regs_user->abi = perf_reg_abi(current);
 		regs_user->regs = regs;
 	} else if (!(current->flags & PF_KTHREAD)) {
-		perf_get_regs_user(regs_user, regs, regs_user_copy);
+		perf_get_regs_user(regs_user, regs);
 	} else {
 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
 		regs_user->regs = NULL;
@@ -7083,8 +7082,7 @@ void perf_prepare_sample(struct perf_eve
 	}
 
 	if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
-		perf_sample_regs_user(&data->regs_user, regs,
-				      &data->regs_user_copy);
+		perf_sample_regs_user(&data->regs_user, regs);
 
 	if (sample_type & PERF_SAMPLE_REGS_USER) {
 		/* regs dump ABI info */
merge bot Applied
#20 lore raw

[tip: perf/urgent] perf/arch: Remove perf_sample_data::regs_user_copy

The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     76a4efa80900fc40e0fdf243b42aec9fb8c35d24
Gitweb:        https://git.kernel.org/tip/76a4efa80900fc40e0fdf243b42aec9fb8c35d24
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 12:14:21 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:34 +01:00

perf/arch: Remove perf_sample_data::regs_user_copy

struct perf_sample_data lives on-stack, we should be careful about it's size. Furthermore, the pt_regs copy in there is only because x86_64 is a trainwreck, solve it differently.

---
 arch/arm/kernel/perf_regs.c   |  3 +--
 arch/arm64/kernel/perf_regs.c |  3 +--
 arch/csky/kernel/perf_regs.c  |  3 +--
 arch/powerpc/perf/perf_regs.c |  3 +--
 arch/riscv/kernel/perf_regs.c |  3 +--
 arch/s390/kernel/perf_regs.c  |  3 +--
 arch/x86/kernel/perf_regs.c   | 15 +++++++++++----
 include/linux/perf_event.h    |  6 ------
 include/linux/perf_regs.h     |  6 ++----
 kernel/events/core.c          |  8 +++-----
 10 files changed, 22 insertions(+), 31 deletions(-)
arch/arm/kernel/perf_regs.c +1 -2
index 05fe92a..0529f90 100644
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/arm64/kernel/perf_regs.c +1 -2
index 94e8718..f6f58e6 100644
@@ -73,8 +73,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/csky/kernel/perf_regs.c +1 -2
index eb32838..09b7f88 100644
@@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/powerpc/perf/perf_regs.c +1 -2
index 8e53f2f..6f681b1 100644
@@ -144,8 +144,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = (regs_user->regs) ? perf_reg_abi(current) :
arch/riscv/kernel/perf_regs.c +1 -2
index 04a38fb..fd304a2 100644
@@ -36,8 +36,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
arch/s390/kernel/perf_regs.c +1 -2
index 4352a50..6e9e5d5 100644
@@ -53,8 +53,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	/*
 	 * Use the regs from the first interruption and let
arch/x86/kernel/perf_regs.c +11 -4
index bb7e113..f9e5352 100644
@@ -101,8 +101,7 @@ u64 perf_reg_abi(struct task_struct *task)
 }
 
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
@@ -129,12 +128,20 @@ u64 perf_reg_abi(struct task_struct *task)
 		return PERF_SAMPLE_REGS_ABI_64;
 }
 
+static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs);
+
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy)
+			struct pt_regs *regs)
 {
+	struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs);
 	struct pt_regs *user_regs = task_pt_regs(current);
 
+	if (!in_nmi()) {
+		regs_user->regs = user_regs;
+		regs_user->abi = perf_reg_abi(current);
+		return;
+	}
+
 	/*
 	 * If we're in an NMI that interrupted task_pt_regs setup, then
 	 * we can't sample user regs at all.  This check isn't really
include/linux/perf_event.h +0 -6
index b775ae0..96450f6 100644
@@ -1022,13 +1022,7 @@ struct perf_sample_data {
 	struct perf_callchain_entry	*callchain;
 	u64				aux_size;
 
-	/*
-	 * regs_user may point to task_pt_regs or to regs_user_copy, depending
-	 * on arch details.
-	 */
 	struct perf_regs		regs_user;
-	struct pt_regs			regs_user_copy;
-
 	struct perf_regs		regs_intr;
 	u64				stack_user_size;
 
include/linux/perf_regs.h +2 -4
index 2d12e97..f632c57 100644
@@ -20,8 +20,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx);
 int perf_reg_validate(u64 mask);
 u64 perf_reg_abi(struct task_struct *task);
 void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs,
-			struct pt_regs *regs_user_copy);
+			struct pt_regs *regs);
 #else
 
 #define PERF_REG_EXTENDED_MASK	0
@@ -42,8 +41,7 @@ static inline u64 perf_reg_abi(struct task_struct *task)
 }
 
 static inline void perf_get_regs_user(struct perf_regs *regs_user,
-				      struct pt_regs *regs,
-				      struct pt_regs *regs_user_copy)
+				      struct pt_regs *regs)
 {
 	regs_user->regs = task_pt_regs(current);
 	regs_user->abi = perf_reg_abi(current);
kernel/events/core.c +3 -5
index fc681c7..d67c9cb 100644
@@ -6374,14 +6374,13 @@ perf_output_sample_regs(struct perf_output_handle *handle,
 }
 
 static void perf_sample_regs_user(struct perf_regs *regs_user,
-				  struct pt_regs *regs,
-				  struct pt_regs *regs_user_copy)
+				  struct pt_regs *regs)
 {
 	if (user_mode(regs)) {
 		regs_user->abi = perf_reg_abi(current);
 		regs_user->regs = regs;
 	} else if (!(current->flags & PF_KTHREAD)) {
-		perf_get_regs_user(regs_user, regs, regs_user_copy);
+		perf_get_regs_user(regs_user, regs);
 	} else {
 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
 		regs_user->regs = NULL;
@@ -7083,8 +7082,7 @@ void perf_prepare_sample(struct perf_event_header *header,
 	}
 
 	if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
-		perf_sample_regs_user(&data->regs_user, regs,
-				      &data->regs_user_copy);
+		perf_sample_regs_user(&data->regs_user, regs);
 
 	if (sample_type & PERF_SAMPLE_REGS_USER) {
 		/* regs dump ABI info */
author maintainer
#21 lore raw

PATCH 6/6 perf/x86: Make dummy_iregs static

In reply to Peter Zijlstra

Having pt_regs on-stack is unfortunate, it's 168 bytes. Since it isn't actually used, make it a static variable. This both gets if off the stack and ensures it gets 0 initialized, just in case someone does look at it.

---
 arch/x86/events/intel/ds.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/events/intel/ds.c +1 -1
@@ -1735,7 +1735,7 @@ static void __intel_pmu_pebs_event(struc
 	struct x86_perf_regs perf_regs;
 	struct pt_regs *regs = &perf_regs.regs;
 	void *at = get_next_pebs_record_by_bit(base, top, bit);
-	struct pt_regs dummy_iregs;
+	static struct pt_regs dummy_iregs;
 
 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
 		/*
merge bot Applied
#22 lore raw

[tip: perf/urgent] perf/x86: Make dummy_iregs static

The following commit has been merged into the perf/urgent branch of tip:

The following commit has been merged into the perf/urgent branch of tip:

Commit-ID:     e506d1dac0edb2df82f2aa0582e814f9cd9aa07d
Gitweb:        https://git.kernel.org/tip/e506d1dac0edb2df82f2aa0582e814f9cd9aa07d
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 30 Oct 2020 12:15:06 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Mon, 09 Nov 2020 18:12:35 +01:00

perf/x86: Make dummy_iregs static

Having pt_regs on-stack is unfortunate, it's 168 bytes. Since it isn't actually used, make it a static variable. This both gets if off the stack and ensures it gets 0 initialized, just in case someone does look at it.

---
 arch/x86/events/intel/ds.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/events/intel/ds.c +1 -1
index 276b29d..b47cc42 100644
@@ -1738,7 +1738,7 @@ __intel_pmu_pebs_event(struct perf_event *event,
 	struct x86_perf_regs perf_regs;
 	struct pt_regs *regs = &perf_regs.regs;
 	void *at = get_next_pebs_record_by_bit(base, top, bit);
-	struct pt_regs dummy_iregs;
+	static struct pt_regs dummy_iregs;
 
 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
 		/*