Skip to contentSkip to conversation

perf/ftrace: Optimize perf/tracepoint interaction for single events

tip:perf/core tip-bot for Peter Zijlstra 3 participants 10 messages May 18, 2010 – May 21, 2010

Subsystems (3)
lore
Share & tools
lore
#0 lore raw
Commit-ID:  4f41c013f553957765902fb01475972f0af3e8e7
Gitweb:     http://git.kernel.org/tip/4f41c013f553957765902fb01475972f0af3e8e7
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Tue, 18 May 2010 18:08:32 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 18 May 2010 18:35:46 +0200

perf/ftrace: Optimize perf/tracepoint interaction for single events

When we've got but a single event per tracepoint there is no reason to try and multiplex it so don't.

Cc Steven Rostedt <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>

LKML-Reference: <new-submission>

---
 include/linux/ftrace_event.h    |    8 +++++---
 include/linux/perf_event.h      |    2 +-
 include/trace/ftrace.h          |    3 ++-
 kernel/perf_event.c             |   15 ++++++++++-----
 kernel/trace/trace_event_perf.c |   11 +++++++----
 kernel/trace/trace_kprobe.c     |    4 ++--
 kernel/trace/trace_syscalls.c   |    6 ++++--
 7 files changed, 31 insertions(+), 18 deletions(-)
include/linux/ftrace_event.h +5 -3
index c0f4b36..c809100 100644
@@ -132,6 +132,7 @@ struct ftrace_event_call {
 	void			*data;
 
 	int			perf_refcount;
+	void			*perf_data;
 	int			(*perf_event_enable)(struct ftrace_event_call *);
 	void			(*perf_event_disable)(struct ftrace_event_call *);
 };
@@ -190,7 +191,7 @@ struct perf_event;
 
 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
 
-extern int perf_trace_enable(int event_id);
+extern int perf_trace_enable(int event_id, void *data);
 extern void perf_trace_disable(int event_id);
 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
@@ -201,11 +202,12 @@ perf_trace_buf_prepare(int size, unsigned short type, int *rctxp,
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, unsigned long irq_flags, struct pt_regs *regs)
+		       u64 count, unsigned long irq_flags, struct pt_regs *regs,
+		       void *event)
 {
 	struct trace_entry *entry = raw_data;
 
-	perf_tp_event(entry->type, addr, count, raw_data, size, regs);
+	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
 	perf_swevent_put_recursion_context(rctx);
 	local_irq_restore(irq_flags);
 }
include/linux/perf_event.h +1 -1
index 3fd5c82..0b521fc 100644
@@ -994,7 +994,7 @@ static inline bool perf_paranoid_kernel(void)
 
 extern void perf_event_init(void);
 extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
-			  int entry_size, struct pt_regs *regs);
+			  int entry_size, struct pt_regs *regs, void *event);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags
include/trace/ftrace.h +2 -1
index 882c648..0a29df0 100644
@@ -785,7 +785,8 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 	{ assign; }							\
 									\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, irq_flags, __regs);		\
+			       __count, irq_flags, __regs, 		\
+			      event_call->perf_data);			\
 }
 
 #undef DEFINE_EVENT
kernel/perf_event.c +10 -5
index a4fa381..17ac47f 100644
@@ -4468,8 +4468,9 @@ static int swevent_hlist_get(struct perf_event *event)
 #ifdef CONFIG_EVENT_TRACING
 
 void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
-		   int entry_size, struct pt_regs *regs)
+		   int entry_size, struct pt_regs *regs, void *event)
 {
+	const int type = PERF_TYPE_TRACEPOINT;
 	struct perf_sample_data data;
 	struct perf_raw_record raw = {
 		.size = entry_size,
@@ -4479,9 +4480,13 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
 	perf_sample_data_init(&data, addr);
 	data.raw = &raw;
 
-	/* Trace events already protected against recursion */
-	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
-			 &data, regs);
+	if (!event) {
+		do_perf_sw_event(type, event_id, count, 1, &data, regs);
+		return;
+	}
+
+	if (perf_swevent_match(event, type, event_id, &data, regs))
+		perf_swevent_add(event, count, 1, &data, regs);
 }
 EXPORT_SYMBOL_GPL(perf_tp_event);
 
@@ -4514,7 +4519,7 @@ static const struct pmu *tp_perf_event_init(struct perf_event *event)
 			!capable(CAP_SYS_ADMIN))
 		return ERR_PTR(-EPERM);
 
-	if (perf_trace_enable(event->attr.config))
+	if (perf_trace_enable(event->attr.config, event))
 		return NULL;
 
 	event->destroy = tp_perf_event_destroy;
kernel/trace/trace_event_perf.c +7 -4
index 0565bb4..89b780a 100644
@@ -27,13 +27,15 @@ typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
 /* Count the events in use (per event id, not per instance) */
 static int	total_ref_count;
 
-static int perf_trace_event_enable(struct ftrace_event_call *event)
+static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
 {
 	char *buf;
 	int ret = -ENOMEM;
 
-	if (event->perf_refcount++ > 0)
+	if (event->perf_refcount++ > 0) {
+		event->perf_data = NULL;
 		return 0;
+	}
 
 	if (!total_ref_count) {
 		buf = (char *)alloc_percpu(perf_trace_t);
@@ -51,6 +53,7 @@ static int perf_trace_event_enable(struct ftrace_event_call *event)
 
 	ret = event->perf_event_enable(event);
 	if (!ret) {
+		event->perf_data = data;
 		total_ref_count++;
 		return 0;
 	}
@@ -68,7 +71,7 @@ fail_buf:
 	return ret;
 }
 
-int perf_trace_enable(int event_id)
+int perf_trace_enable(int event_id, void *data)
 {
 	struct ftrace_event_call *event;
 	int ret = -EINVAL;
@@ -77,7 +80,7 @@ int perf_trace_enable(int event_id)
 	list_for_each_entry(event, &ftrace_events, list) {
 		if (event->id == event_id && event->perf_event_enable &&
 		    try_module_get(event->mod)) {
-			ret = perf_trace_event_enable(event);
+			ret = perf_trace_event_enable(event, data);
 			break;
 		}
 	}
kernel/trace/trace_kprobe.c +2 -2
index a751432..2d7bf41 100644
@@ -1362,7 +1362,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs);
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
 }
 
 /* Kretprobe profile handler */
@@ -1395,7 +1395,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			       irq_flags, regs);
+			       irq_flags, regs, call->perf_data);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +4 -2
index 4d6d711..9eff1a4 100644
@@ -468,7 +468,8 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+			sys_data->enter_event->perf_data);
 }
 
 int perf_sysenter_enable(struct ftrace_event_call *call)
@@ -543,7 +544,8 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+			sys_data->exit_event->perf_data);
 }
 
 int perf_sysexit_enable(struct ftrace_event_call *call)
#1 lore raw
Quoted 19 lines · tip-bot for Peter Zijlstra
Commit-ID:  4f41c013f553957765902fb01475972f0af3e8e7
Gitweb:     http://git.kernel.org/tip/4f41c013f553957765902fb01475972f0af3e8e7
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Tue, 18 May 2010 18:08:32 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 18 May 2010 18:35:46 +0200

perf/ftrace: Optimize perf/tracepoint interaction for single events

When we've got but a single event per tracepoint there is no reason to try and multiplex it so don't.

Cc Steven Rostedt <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Arnaldo Carvalho de Melo <[email protected]>
Quoted 3 lines

LKML-Reference: <new-submission>

---

I haven't seen this patch going.

But yeah that's a nice optimization for UP machines.

maintainer
#2 lore raw
Quoted 10 lines · Frederic Weisbecker
Quoted 9 lines · tip-bot for Peter Zijlstra
Commit-ID:  4f41c013f553957765902fb01475972f0af3e8e7
Gitweb:     http://git.kernel.org/tip/4f41c013f553957765902fb01475972f0af3e8e7
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Tue, 18 May 2010 18:08:32 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 18 May 2010 18:35:46 +0200

perf/ftrace: Optimize perf/tracepoint interaction for single events

Quoted 7 lines
Quoted 3 lines

When we've got but a single event per tracepoint there is no reason to try and multiplex it so don't.

I haven't seen this patch going.

But yeah that's a nice optimization for UP machines.

Uhm, doesn't have anything to do with UP machines... it works quite well on SMP as well.

Its about tracepoints with a single event attached.

#3 lore raw
Quoted 15 lines · Peter Zijlstra
Quoted 13 lines · Frederic Weisbecker
Quoted 12 lines · tip-bot for Peter Zijlstra
Commit-ID:  4f41c013f553957765902fb01475972f0af3e8e7
Gitweb:     http://git.kernel.org/tip/4f41c013f553957765902fb01475972f0af3e8e7
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Tue, 18 May 2010 18:08:32 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 18 May 2010 18:35:46 +0200

perf/ftrace: Optimize perf/tracepoint interaction for single events

When we've got but a single event per tracepoint there is no reason to try and multiplex it so don't.

Quoted 8 lines
Quoted 3 lines

I haven't seen this patch going.

But yeah that's a nice optimization for UP machines.

Uhm, doesn't have anything to do with UP machines... it works quite well on SMP as well.

Its about tracepoints with a single event attached.

Yeah, then if you launch perf sched for example, you'll have NR_CPU perf events attached to each lock trace events, since we create one event per cpu.

Or may be I'm somewhat confused...

maintainer
#4 lore raw
Quoted 2 lines · Frederic Weisbecker

Yeah, then if you launch perf sched for example, you'll have NR_CPU perf events attached to each lock trace events, since we create one event per cpu.

Right, but simple per task (non-inherited) would have only 1.

But yeah, plenty of room to improve here.

maintainer
#5 lore raw
In reply to Frederic Weisbecker

How about something like the below?

---
Subject: perf, trace: Remove IRQ-disable from perf/tracepoint interaction
From: Peter Zijlstra <[email protected]>
Date: Wed May 19 10:52:27 CEST 2010

---
 include/linux/ftrace_event.h    |    9 +----
 include/trace/ftrace.h          |   17 ++++------
 kernel/trace/trace_event_perf.c |   67 ++++++++++++++++------------------------
 kernel/trace/trace_kprobe.c     |   10 ++---
 kernel/trace/trace_syscalls.c   |   10 ++---
 5 files changed, 46 insertions(+), 67 deletions(-)
include/linux/ftrace_event.h +3 -6
Index: linux-2.6/include/linux/ftrace_event.h
===================================================================
@@ -197,20 +197,17 @@ extern void perf_trace_disable(int event
 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
-extern void *
-perf_trace_buf_prepare(int size, unsigned short type, int *rctxp,
-			 unsigned long *irq_flags);
+extern void *perf_trace_buf_prepare(int size, unsigned short type,
+				    struct pt_regs *regs, int *rctxp);
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, unsigned long irq_flags, struct pt_regs *regs,
-		       void *event)
+		       u64 count, struct pt_regs *regs, void *event)
 {
 	struct trace_entry *entry = raw_data;
 
 	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
 	perf_swevent_put_recursion_context(rctx);
-	local_irq_restore(irq_flags);
 }
 #endif
 
include/trace/ftrace.h +7 -10
Index: linux-2.6/include/trace/ftrace.h
===================================================================
@@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_ev
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
-	unsigned long irq_flags;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\
@@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_ev
 	if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE,		\
 		      "profile buffer not large enough"))		\
 		return;							\
+									\
 	entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare(	\
-		__entry_size, event_call->id, &rctx, &irq_flags);	\
+		__entry_size, event_call->id, __regs, &rctx);		\
 	if (!entry)							\
 		return;							\
+									\
 	tstruct								\
 									\
 	{ assign; }							\
 									\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, irq_flags, __regs, 		\
-			      event_call->perf_data);			\
+			       __count, __regs, event_call->perf_data);	\
 }
 
 #undef DEFINE_EVENT
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev
 static notrace void perf_trace_##call(proto)				\
 {									\
 	struct ftrace_event_call *event_call = &event_##call;		\
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-									\
-	perf_fetch_caller_regs(__regs, 1);				\
-									\
-	perf_trace_templ_##template(event_call, __regs, args);		\
+	struct pt_regs __regs;						\
 									\
-	put_cpu_var(perf_trace_regs);					\
+	perf_fetch_caller_regs(&__regs, 1);				\
+	perf_trace_templ_##template(event_call, &__regs, args);		\
 }
 
 #undef DEFINE_EVENT_PRINT
kernel/trace/trace_event_perf.c +28 -39

134 lines not on this page · raw message

kernel/trace/trace_kprobe.c +4 -6
Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(s
 	struct kprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(s
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
 }
 
 /* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_fun
 	struct kretprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_fun
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_fun
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			       irq_flags, regs, call->perf_data);
+			      regs, call->perf_data);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +4 -6
Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt
 		return;
 
 	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
-				sys_data->enter_event->id, &rctx, &flags);
+				sys_data->enter_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->enter_event->perf_data);
 }
 
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_
 		return;
 
 	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
-				sys_data->exit_event->id, &rctx, &flags);
+				sys_data->exit_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->exit_event->perf_data);
 }
 
maintainer
#7 lore raw
Quoted 11 lines · Frederic Weisbecker
Quoted 10 lines · Peter Zijlstra

How about something like the below?

---
Subject: perf, trace: Remove IRQ-disable from perf/tracepoint interaction
From: Peter Zijlstra <[email protected]>
Date: Wed May 19 10:52:27 CEST 2010

Quoted 6 lines
Quoted 2 lines

---

Looks fine if we don't have nested irqs.

Well, I was thinking the per-context recursion thing would sort that out. We can have 1 event per context in-flight.

#8 lore raw
Quoted 17 lines · Peter Zijlstra
Quoted 16 lines · Frederic Weisbecker
Quoted 12 lines · Peter Zijlstra

How about something like the below?

---
Subject: perf, trace: Remove IRQ-disable from perf/tracepoint interaction
From: Peter Zijlstra <[email protected]>
Date: Wed May 19 10:52:27 CEST 2010

---

Quoted 4 lines
Quoted 1 line

Looks fine if we don't have nested irqs.

Well, I was thinking the per-context recursion thing would sort that out. We can have 1 event per context in-flight.

Ah good point.

#9 lore raw

[tip:perf/core] perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

In reply to Peter Zijlstra
Commit-ID:  b7e2ecef92d2e7785e6d76b41e5ba8bcbc45259d
Gitweb:     http://git.kernel.org/tip/b7e2ecef92d2e7785e6d76b41e5ba8bcbc45259d
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Wed, 19 May 2010 10:52:27 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:56 +0200

perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

Improves performance.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <1274259525.5605.10352.camel@twins>

---
 include/linux/ftrace_event.h    |    9 ++---
 include/trace/ftrace.h          |   17 ++++-----
 kernel/trace/trace_event_perf.c |   73 +++++++++++++++-----------------------
 kernel/trace/trace_kprobe.c     |   10 ++---
 kernel/trace/trace_syscalls.c   |   10 ++---
 5 files changed, 47 insertions(+), 72 deletions(-)
include/linux/ftrace_event.h +3 -6
index a9775dd..126071b 100644
@@ -197,20 +197,17 @@ extern void perf_trace_disable(int event_id);
 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
-extern void *
-perf_trace_buf_prepare(int size, unsigned short type, int *rctxp,
-			 unsigned long *irq_flags);
+extern void *perf_trace_buf_prepare(int size, unsigned short type,
+				    struct pt_regs *regs, int *rctxp);
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, unsigned long irq_flags, struct pt_regs *regs,
-		       void *event)
+		       u64 count, struct pt_regs *regs, void *event)
 {
 	struct trace_entry *entry = raw_data;
 
 	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
 	perf_swevent_put_recursion_context(rctx);
-	local_irq_restore(irq_flags);
 }
 #endif
 
include/trace/ftrace.h +7 -10
index 1016b21..f282885 100644
@@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
-	unsigned long irq_flags;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\
@@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 	if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE,		\
 		      "profile buffer not large enough"))		\
 		return;							\
+									\
 	entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare(	\
-		__entry_size, event_call->id, &rctx, &irq_flags);	\
+		__entry_size, event_call->id, __regs, &rctx);		\
 	if (!entry)							\
 		return;							\
+									\
 	tstruct								\
 									\
 	{ assign; }							\
 									\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, irq_flags, __regs, 		\
-			      event_call->perf_data);			\
+			       __count, __regs, event_call->perf_data);	\
 }
 
 #undef DEFINE_EVENT
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 static notrace void perf_trace_##call(proto)				\
 {									\
 	struct ftrace_event_call *event_call = &event_##call;		\
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-									\
-	perf_fetch_caller_regs(__regs, 1);				\
-									\
-	perf_trace_templ_##template(event_call, __regs, args);		\
+	struct pt_regs __regs;						\
 									\
-	put_cpu_var(perf_trace_regs);					\
+	perf_fetch_caller_regs(&__regs, 1);				\
+	perf_trace_templ_##template(event_call, &__regs, args);		\
 }
 
 #undef DEFINE_EVENT_PRINT
kernel/trace/trace_event_perf.c +29 -44

154 lines not on this page · raw message

kernel/trace/trace_kprobe.c +4 -6
index 2d7bf41..20c96de 100644
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	struct kprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
 }
 
 /* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 	struct kretprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			       irq_flags, regs, call->perf_data);
+			      regs, call->perf_data);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +4 -6
index 9eff1a4..a657cef 100644
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
 		return;
 
 	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
-				sys_data->enter_event->id, &rctx, &flags);
+				sys_data->enter_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->enter_event->perf_data);
 }
 
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
 		return;
 
 	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
-				sys_data->exit_event->id, &rctx, &flags);
+				sys_data->exit_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->exit_event->perf_data);
 }