Skip to contentSkip to conversation

perf tracepoint and output optimizations

PATCH In review Peter Zijlstra 7 participants 53 messages May 21, 2010 – May 31, 2010

Subsystems (4)
lore
Share & tools
lore
#51 lore raw

PATCH 10/10 perf: Optimize perf_tp_event_match

In reply to Peter Zijlstra

Since we know tracepoints come from kernel context, avoid conditionals that try and establish that very fact.

---
 kernel/perf_event.c |    7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/perf_event.c +4 -1
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -4477,7 +4477,10 @@ static int perf_tp_event_match(struct pe
 				struct perf_sample_data *data,
 				struct pt_regs *regs)
 {
-	if (perf_exclude_event(event, regs))
+	/*
+	 * All tracepoints are from kernel-space.
+	 */
+	if (event->attr.exclude_kernel)
 		return 0;
 
 	if (!perf_tp_filter_match(event, data))
#35 lore raw

PATCH 4/10 perf-record: Remove -M

In reply to Peter Zijlstra

Since it is not allowed to create cross-cpu (or cross-task) buffers, this option is no longer valid.

---
 tools/perf/builtin-record.c |   36 ++++++++++++------------------------
 1 file changed, 12 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +12 -24
Index: linux-2.6/tools/perf/builtin-record.c
===================================================================
@@ -61,8 +61,6 @@ static bool			call_graph			=  false;
 static bool			inherit_stat			=  false;
 static bool			no_samples			=  false;
 static bool			sample_address			=  false;
-static bool			multiplex			=  false;
-static int			multiplex_fd			=     -1;
 
 static long			samples				=      0;
 static struct timeval		last_read;
@@ -381,27 +379,19 @@ try_again:
 		 */
 		if (group && group_fd == -1)
 			group_fd = fd[nr_cpu][counter][thread_index];
-		if (multiplex && multiplex_fd == -1)
-			multiplex_fd = fd[nr_cpu][counter][thread_index];
 
-		if (multiplex && fd[nr_cpu][counter][thread_index] != multiplex_fd) {
-
-			ret = ioctl(fd[nr_cpu][counter][thread_index], PERF_EVENT_IOC_SET_OUTPUT, multiplex_fd);
-			assert(ret != -1);
-		} else {
-			event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
-			event_array[nr_poll].events = POLLIN;
-			nr_poll++;
-
-			mmap_array[nr_cpu][counter][thread_index].counter = counter;
-			mmap_array[nr_cpu][counter][thread_index].prev = 0;
-			mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
-			mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
-				PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
-			if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
-				error("failed to mmap with %d (%s)\n", errno, strerror(errno));
-				exit(-1);
-			}
+		event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
+		event_array[nr_poll].events = POLLIN;
+		nr_poll++;
+
+		mmap_array[nr_cpu][counter][thread_index].counter = counter;
+		mmap_array[nr_cpu][counter][thread_index].prev = 0;
+		mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
+		mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
+			PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
+		if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
+			error("failed to mmap with %d (%s)\n", errno, strerror(errno));
+			exit(-1);
 		}
 
 		if (filter != NULL) {
@@ -835,8 +825,6 @@ static const struct option options[] = {
 		    "Sample addresses"),
 	OPT_BOOLEAN('n', "no-samples", &no_samples,
 		    "don't sample"),
-	OPT_BOOLEAN('M', "multiplex", &multiplex,
-		    "multiplex counter output in a single channel"),
 	OPT_END()
 };
 
#47 lore raw

PATCH 9/10 perf: Remove more fastpath code

In reply to Peter Zijlstra

Sanity checks cost instructions

---
 include/linux/perf_event.h |    2 --
 kernel/perf_event.c        |   20 ++++----------------
 2 files changed, 4 insertions(+), 18 deletions(-)
include/linux/perf_event.h +0 -2
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -803,8 +803,6 @@ struct perf_cpu_context {
 struct perf_output_handle {
 	struct perf_event		*event;
 	struct perf_mmap_data		*data;
-	unsigned long			head;
-	unsigned long			offset;
 	unsigned long			wakeup;
 	unsigned long			size;
 	void				*addr;
kernel/perf_event.c +4 -16
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -2966,20 +2966,11 @@ again:
 	preempt_enable();
 }
 
-void perf_output_copy(struct perf_output_handle *handle,
+__always_inline void perf_output_copy(struct perf_output_handle *handle,
 		      const void *buf, unsigned int len)
 {
-	handle->offset += len;
-
-	/*
-	 * Check we didn't copy past our reservation window, taking the
-	 * possible unsigned int wrap into account.
-	 */
-	if (WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0))
-		return;
-
 	do {
-		unsigned long size = min(handle->size, len);
+		unsigned long size = min_t(unsigned long, handle->size, len);
 
 		memcpy(handle->addr, buf, size);
 
@@ -3054,15 +3045,12 @@ int perf_output_begin(struct perf_output
 			goto fail;
 	} while (local_cmpxchg(&data->head, offset, head) != offset);
 
-	handle->offset	= offset;
-	handle->head	= head;
-
 	if (head - local_read(&data->wakeup) > data->watermark)
 		local_add(data->watermark, &data->wakeup);
 
-	handle->page = handle->offset >> (PAGE_SHIFT + page_order(data));
+	handle->page = offset >> (PAGE_SHIFT + page_order(data));
 	handle->page &= data->nr_pages - 1;
-	handle->size = handle->offset & ((PAGE_SIZE << page_order(data)) - 1);
+	handle->size = offset & ((PAGE_SIZE << page_order(data)) - 1);
 	handle->addr = data->data_pages[handle->page];
 	handle->addr += handle->size;
 	handle->size = (PAGE_SIZE << page_order(data)) - handle->size;
#43 lore raw

PATCH 7/10 perf: Optimize perf_output_copy

In reply to Peter Zijlstra

Reduce the clutter in perf_output_copy() by keeping an interator in perf_output_handle.

---
 include/linux/perf_event.h |    3 ++
 kernel/perf_event.c        |   54 +++++++++++++++++++++------------------------
 2 files changed, 29 insertions(+), 28 deletions(-)
include/linux/perf_event.h +3 -0
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -806,6 +806,9 @@ struct perf_output_handle {
 	unsigned long			head;
 	unsigned long			offset;
 	unsigned long			wakeup;
+	unsigned long			size;
+	void				*addr;
+	int				page;
 	int				nmi;
 	int				sample;
 };
kernel/perf_event.c +26 -28

71 lines not on this page · raw message

#33 lore raw

PATCH 3/10 perf: Ensure IOC_OUTPUT isnt used to create multi-writer buffers

In reply to Peter Zijlstra

Since we want to ensure buffers only have a single writer, we must avoid creating one with multiple.

---
 kernel/perf_event.c |   19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
kernel/perf_event.c +19 -0
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -4885,6 +4885,13 @@ static int perf_event_set_output(struct 
 	int fput_needed = 0;
 	int ret = -EINVAL;
 
+	/*
+	 * Don't allow output of inherited per-task events. This would
+	 * create performance issues due to cross cpu access.
+	 */
+	if (event->cpu == -1 && event->attr.inherit)
+		return -EINVAL;
+
 	if (!output_fd)
 		goto set;
 
@@ -4905,6 +4912,18 @@ static int perf_event_set_output(struct 
 	if (event->data)
 		goto out;
 
+	/*
+	 * Don't allow cross-cpu buffers
+	 */
+	if (output_event->cpu != event->cpu)
+		goto out;
+
+	/*
+	 * If its not a per-cpu buffer, it must be the same task.
+	 */
+	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
+		goto out;
+
 	atomic_long_inc(&output_file->f_count);
 
 set:
#0 lore raw

cover perf tracepoint and output optimizations

A number of patches that reworks the perf-tracepoint infrastructure and optimizes some of the generic perf output paths.

#45 lore raw

PATCH 8/10 perf: Optimize the !vmalloc backed buffer

Reduce code and data by using the knowledge that for !PERF_USE_VMALLOC data_order is always 0.

---
 include/linux/perf_event.h |    2 +-
 kernel/perf_event.c        |   41 ++++++++++++++++++++++++++---------------
 2 files changed, 27 insertions(+), 16 deletions(-)
include/linux/perf_event.h +1 -1
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -588,8 +588,8 @@ struct perf_mmap_data {
 	struct rcu_head			rcu_head;
 #ifdef CONFIG_PERF_USE_VMALLOC
 	struct work_struct		work;
+	int				page_order;	/* allocation order  */
 #endif
-	int				data_order;	/* allocation order  */
 	int				nr_pages;	/* nr of data pages  */
 	int				writable;	/* are we writable   */
 	int				nr_locked;	/* nr pages mlocked  */
kernel/perf_event.c +26 -15

102 lines not on this page · raw message

#1 lore raw

PATCH 1/10 perf, trace: Remove IRQ-disable from perf/tracepoint interaction

In reply to Peter Zijlstra

LKML-Reference: <1274259525.5605.10352.camel@twins>

---
 include/linux/ftrace_event.h    |    9 +---
 include/trace/ftrace.h          |   17 +++------
 kernel/trace/trace_event_perf.c |   73 +++++++++++++++-------------------------
 kernel/trace/trace_kprobe.c     |   10 ++---
 kernel/trace/trace_syscalls.c   |   10 ++---
 5 files changed, 47 insertions(+), 72 deletions(-)
include/linux/ftrace_event.h +3 -6
Index: linux-2.6/include/linux/ftrace_event.h
===================================================================
@@ -197,20 +197,17 @@ extern void perf_trace_disable(int event
 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
-extern void *
-perf_trace_buf_prepare(int size, unsigned short type, int *rctxp,
-			 unsigned long *irq_flags);
+extern void *perf_trace_buf_prepare(int size, unsigned short type,
+				    struct pt_regs *regs, int *rctxp);
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, unsigned long irq_flags, struct pt_regs *regs,
-		       void *event)
+		       u64 count, struct pt_regs *regs, void *event)
 {
 	struct trace_entry *entry = raw_data;
 
 	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
 	perf_swevent_put_recursion_context(rctx);
-	local_irq_restore(irq_flags);
 }
 #endif
 
include/trace/ftrace.h +7 -10
Index: linux-2.6/include/trace/ftrace.h
===================================================================
@@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_ev
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
-	unsigned long irq_flags;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\
@@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_ev
 	if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE,		\
 		      "profile buffer not large enough"))		\
 		return;							\
+									\
 	entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare(	\
-		__entry_size, event_call->id, &rctx, &irq_flags);	\
+		__entry_size, event_call->id, __regs, &rctx);		\
 	if (!entry)							\
 		return;							\
+									\
 	tstruct								\
 									\
 	{ assign; }							\
 									\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, irq_flags, __regs, 		\
-			      event_call->perf_data);			\
+			       __count, __regs, event_call->perf_data);	\
 }
 
 #undef DEFINE_EVENT
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev
 static notrace void perf_trace_##call(proto)				\
 {									\
 	struct ftrace_event_call *event_call = &event_##call;		\
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-									\
-	perf_fetch_caller_regs(__regs, 1);				\
-									\
-	perf_trace_templ_##template(event_call, __regs, args);		\
+	struct pt_regs __regs;						\
 									\
-	put_cpu_var(perf_trace_regs);					\
+	perf_fetch_caller_regs(&__regs, 1);				\
+	perf_trace_templ_##template(event_call, &__regs, args);		\
 }
 
 #undef DEFINE_EVENT_PRINT
kernel/trace/trace_event_perf.c +29 -44

154 lines not on this page · raw message

kernel/trace/trace_kprobe.c +4 -6
Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(s
 	struct kprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(s
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
 }
 
 /* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_fun
 	struct kretprobe_trace_entry_head *entry;
 	u8 *data;
 	int size, __size, i;
-	unsigned long irq_flags;
 	int rctx;
 
 	__size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_fun
 		     "profile buffer not large enough"))
 		return;
 
-	entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
+	entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
 	if (!entry)
 		return;
 
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_fun
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			       irq_flags, regs, call->perf_data);
+			      regs, call->perf_data);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +4 -6
Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt
 		return;
 
 	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
-				sys_data->enter_event->id, &rctx, &flags);
+				sys_data->enter_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->enter_event->perf_data);
 }
 
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
-	unsigned long flags;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_
 		return;
 
 	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
-				sys_data->exit_event->id, &rctx, &flags);
+				sys_data->exit_event->id, regs, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
 			sys_data->exit_event->perf_data);
 }
 
#41 lore raw

PATCH 6/10 perf: Fix wakeup storm for RO mmap()s

In reply to Peter Zijlstra

RO mmap()s don't update the tail pointer, so comparing against it for determining the written data size doesn't really do any good.

Keep track of when we last did a wakeup, and compare against that.

---
 include/linux/perf_event.h |    2 +-
 kernel/perf_event.c        |    4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
include/linux/perf_event.h +1 -1
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -599,7 +599,7 @@ struct perf_mmap_data {
 	local_t				head;		/* write position    */
 	local_t				nest;		/* nested writers    */
 	local_t				events;		/* event limit       */
-	local_t				wakeup;		/* needs a wakeup    */
+	local_t				wakeup;		/* wakeup stamp      */
 	local_t				lost;		/* nr records lost   */
 
 	long				watermark;	/* wakeup watermark  */
kernel/perf_event.c +2 -2
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -3055,8 +3055,8 @@ int perf_output_begin(struct perf_output
 	handle->offset	= offset;
 	handle->head	= head;
 
-	if (head - tail > data->watermark)
-		local_inc(&data->wakeup);
+	if (head - local_read(&data->wakeup) > data->watermark)
+		local_add(data->watermark, &data->wakeup);
 
 	if (have_lost) {
 		lost_event.header.type = PERF_RECORD_LOST;
#13 lore raw

PATCH 2/10 perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra

Avoid the swevent hash-table by using per-tracepoint hlists.

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

---
 include/linux/ftrace_event.h    |   16 ++---
 include/linux/perf_event.h      |    6 +
 include/trace/ftrace.h          |    4 -
 kernel/perf_event.c             |   94 +++++++++++++++--------------
 kernel/trace/trace_event_perf.c |  127 +++++++++++++++++++++-------------------
 kernel/trace/trace_kprobe.c     |    9 +-
 kernel/trace/trace_syscalls.c   |   11 ++-
 7 files changed, 143 insertions(+), 124 deletions(-)
include/linux/ftrace_event.h +8 -8
Index: linux-2.6/include/linux/ftrace_event.h
===================================================================
@@ -133,7 +133,7 @@ struct ftrace_event_call {
 	void			*data;
 
 	int			perf_refcount;
-	void			*perf_data;
+	struct hlist_head	*perf_events;
 	int			(*perf_event_enable)(struct ftrace_event_call *);
 	void			(*perf_event_disable)(struct ftrace_event_call *);
 };
@@ -192,9 +192,11 @@ struct perf_event;
 
 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
 
-extern int perf_trace_enable(int event_id, void *data);
-extern void perf_trace_disable(int event_id);
-extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
+extern int  perf_trace_init(struct perf_event *event);
+extern void perf_trace_destroy(struct perf_event *event);
+extern int  perf_trace_enable(struct perf_event *event);
+extern void perf_trace_disable(struct perf_event *event);
+extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
 extern void *perf_trace_buf_prepare(int size, unsigned short type,
@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(int 
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, struct pt_regs *regs, void *event)
+		       u64 count, struct pt_regs *regs, void *head)
 {
-	struct trace_entry *entry = raw_data;
-
-	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
+	perf_tp_event(addr, count, raw_data, size, regs, head);
 	perf_swevent_put_recursion_context(rctx);
 }
 #endif
include/trace/ftrace.h +3 -1
Index: linux-2.6/include/trace/ftrace.h
===================================================================
@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_ev
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
+	struct hlist_head *head;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\
@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_ev
 									\
 	{ assign; }							\
 									\
+	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, __regs, event_call->perf_data);	\
+		__count, __regs, head);					\
 }
 
 #undef DEFINE_EVENT
kernel/trace/trace_event_perf.c +67 -60

197 lines not on this page · raw message

kernel/trace/trace_kprobe.c +6 -3
Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================
@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(s
 	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;
@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
 /* Kretprobe profile handler */
@@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_fun
 	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kretprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;
@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_fun
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			      regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +7 -4
Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================
@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->enter_event->perf_data);
+
+	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysenter_enable(struct ftrace_event_call *call)
@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->exit_event->perf_data);
+	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysexit_enable(struct ftrace_event_call *call)
kernel/perf_event.c +48 -46

173 lines not on this page · raw message

include/linux/perf_event.h +4 -2
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -727,6 +727,7 @@ struct perf_event {
 	perf_overflow_handler_t		overflow_handler;
 
 #ifdef CONFIG_EVENT_TRACING
+	struct ftrace_event_call	*tp_event;
 	struct event_filter		*filter;
 #endif
 
@@ -992,8 +993,9 @@ static inline bool perf_paranoid_kernel(
 }
 
 extern void perf_event_init(void);
-extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
-			  int entry_size, struct pt_regs *regs, void *event);
+extern void perf_tp_event(u64 addr, u64 count, void *record,
+			  int entry_size, struct pt_regs *regs,
+			  struct hlist_head *head);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags
#37 lore raw

PATCH 5/10 perf-record: Share per-cpu buffers

In reply to Peter Zijlstra

It seems a waste of space to create a buffer per event, share it per-cpu.

---
 tools/perf/builtin-record.c |   52 +++++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +28 -24
Index: linux-2.6/tools/perf/builtin-record.c
===================================================================
@@ -85,7 +85,7 @@ struct mmap_data {
 	unsigned int		prev;
 };
 
-static struct mmap_data		*mmap_array[MAX_NR_CPUS][MAX_COUNTERS];
+static struct mmap_data		mmap_array[MAX_NR_CPUS];
 
 static unsigned long mmap_read_head(struct mmap_data *md)
 {
@@ -380,18 +380,29 @@ try_again:
 		if (group && group_fd == -1)
 			group_fd = fd[nr_cpu][counter][thread_index];
 
-		event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
-		event_array[nr_poll].events = POLLIN;
-		nr_poll++;
-
-		mmap_array[nr_cpu][counter][thread_index].counter = counter;
-		mmap_array[nr_cpu][counter][thread_index].prev = 0;
-		mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
-		mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
-			PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
-		if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
-			error("failed to mmap with %d (%s)\n", errno, strerror(errno));
-			exit(-1);
+		if (counter || thread_index) {
+			ret = ioctl(fd[nr_cpu][counter][thread_index],
+					PERF_EVENT_IOC_SET_OUTPUT,
+					fd[nr_cpu][0][0]);
+			if (ret) {
+				error("failed to set output: %d (%s)\n", errno,
+						strerror(errno));
+				exit(-1);
+			}
+		} else {
+			mmap_array[nr_cpu].counter = counter;
+			mmap_array[nr_cpu].prev = 0;
+			mmap_array[nr_cpu].mask = mmap_pages*page_size - 1;
+			mmap_array[nr_cpu].base = mmap(NULL, (mmap_pages+1)*page_size,
+				PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
+			if (mmap_array[nr_cpu].base == MAP_FAILED) {
+				error("failed to mmap with %d (%s)\n", errno, strerror(errno));
+				exit(-1);
+			}
+
+			event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
+			event_array[nr_poll].events = POLLIN;
+			nr_poll++;
 		}
 
 		if (filter != NULL) {
@@ -492,16 +503,11 @@ static struct perf_event_header finished
 
 static void mmap_read_all(void)
 {
-	int i, counter, thread;
+	int i;
 
 	for (i = 0; i < nr_cpu; i++) {
-		for (counter = 0; counter < nr_counters; counter++) {
-			for (thread = 0; thread < thread_num; thread++) {
-				if (mmap_array[i][counter][thread].base)
-					mmap_read(&mmap_array[i][counter][thread]);
-			}
-
-		}
+		if (mmap_array[i].base)
+			mmap_read(&mmap_array[i]);
 	}
 
 	if (perf_header__has_feat(&session->header, HEADER_TRACE_INFO))
@@ -876,9 +882,7 @@ int cmd_record(int argc, const char **ar
 	for (i = 0; i < MAX_NR_CPUS; i++) {
 		for (j = 0; j < MAX_COUNTERS; j++) {
 			fd[i][j] = malloc(sizeof(int)*thread_num);
-			mmap_array[i][j] = zalloc(
-				sizeof(struct mmap_data)*thread_num);
-			if (!fd[i][j] || !mmap_array[i][j])
+			if (!fd[i][j])
 				return -ENOMEM;
 		}
 	}
#14 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra
Quoted 71 lines · Peter Zijlstra

Avoid the swevent hash-table by using per-tracepoint hlists.

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

---
 include/linux/ftrace_event.h    |   16 ++---
 include/linux/perf_event.h      |    6 +
 include/trace/ftrace.h          |    4 -
 kernel/perf_event.c             |   94 +++++++++++++++--------------
 kernel/trace/trace_event_perf.c |  127 +++++++++++++++++++++-------------------
 kernel/trace/trace_kprobe.c     |    9 +-
 kernel/trace/trace_syscalls.c   |   11 ++-
 7 files changed, 143 insertions(+), 124 deletions(-)

Index: linux-2.6/include/linux/ftrace_event.h
===================================================================

include/linux/ftrace_event.h +8 -8
@@ -133,7 +133,7 @@ struct ftrace_event_call {
 	void			*data;
 
 	int			perf_refcount;
-	void			*perf_data;
+	struct hlist_head	*perf_events;
 	int			(*perf_event_enable)(struct ftrace_event_call *);
 	void			(*perf_event_disable)(struct ftrace_event_call *);
 };
@@ -192,9 +192,11 @@ struct perf_event;
 
 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
 
-extern int perf_trace_enable(int event_id, void *data);
-extern void perf_trace_disable(int event_id);
-extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
+extern int  perf_trace_init(struct perf_event *event);
+extern void perf_trace_destroy(struct perf_event *event);
+extern int  perf_trace_enable(struct perf_event *event);
+extern void perf_trace_disable(struct perf_event *event);
+extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
 extern void *perf_trace_buf_prepare(int size, unsigned short type,
@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(int 
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, struct pt_regs *regs, void *event)
+		       u64 count, struct pt_regs *regs, void *head)
 {
-	struct trace_entry *entry = raw_data;
-
-	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
+	perf_tp_event(addr, count, raw_data, size, regs, head);
 	perf_swevent_put_recursion_context(rctx);
 }
 #endif

Index: linux-2.6/include/trace/ftrace.h
===================================================================

include/trace/ftrace.h +1 -0
@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_ev
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
+	struct hlist_head *head;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794

Quoted 4 lines
include/trace/ftrace.h +1 -0
@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_ev
 									\
 	{ assign; }							\
 									\
+	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\

Should be rcu_dereference_sched ?

Quoted 27 lines
include/trace/ftrace.h +1 -1
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, __regs, event_call->perf_data);	\
+		__count, __regs, head);					\
 }
 
 #undef DEFINE_EVENT

Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================

kernel/trace/trace_event_perf.c +6 -3
@@ -23,14 +23,25 @@ typedef typeof(unsigned long [PERF_MAX_T
 /* Count the events in use (per event id, not per instance) */
 static int	total_ref_count;
 
-static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
+static int perf_trace_event_init(struct ftrace_event_call *tp_event,
+				 struct perf_event *p_event)
 {
+	struct hlist_head *list;
 	int ret = -ENOMEM;
+	int cpu;
 
-	if (event->perf_refcount++ > 0) {
-		event->perf_data = NULL;
+	p_event->tp_event = tp_event;
+	if (tp_event->perf_refcount++ > 0)
 		return 0;

kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44

Quoted 10 lines
kernel/trace/trace_event_perf.c +9 -1
-	}
+
+	list = alloc_percpu(struct hlist_head);
+	if (!list)
+		goto fail;
+
+	for_each_possible_cpu(cpu)
+		INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
+
+	tp_event->perf_events = list;

I suspect this must be rcu_assign_pointer.

Quoted 84 lines
kernel/trace/trace_event_perf.c +27 -20
 	if (!total_ref_count) {
 		char *buf;
@@ -39,20 +50,20 @@ static int perf_trace_event_enable(struc
 		for (i = 0; i < 4; i++) {
 			buf = (char *)alloc_percpu(perf_trace_t);
 			if (!buf)
-				goto fail_buf;
+				goto fail;
 
-			rcu_assign_pointer(perf_trace_buf[i], buf);
+			perf_trace_buf[i] = buf;
 		}
 	}
 
-	ret = event->perf_event_enable(event);
-	if (!ret) {
-		event->perf_data = data;
-		total_ref_count++;
-		return 0;
-	}
+	ret = tp_event->perf_event_enable(tp_event);
+	if (ret)
+		goto fail;
 
-fail_buf:
+	total_ref_count++;
+	return 0;
+
+fail:
 	if (!total_ref_count) {
 		int i;
 
@@ -61,21 +72,26 @@ fail_buf:
 			perf_trace_buf[i] = NULL;
 		}
 	}
-	event->perf_refcount--;
+
+	if (!--tp_event->perf_refcount) {
+		free_percpu(tp_event->perf_events);
+		tp_event->perf_events = NULL;
+	}
 
 	return ret;
 }
 
-int perf_trace_enable(int event_id, void *data)
+int perf_trace_init(struct perf_event *p_event)
 {
-	struct ftrace_event_call *event;
+	struct ftrace_event_call *tp_event;
+	int event_id = p_event->attr.config;
 	int ret = -EINVAL;
 
 	mutex_lock(&event_mutex);
-	list_for_each_entry(event, &ftrace_events, list) {
-		if (event->id == event_id && event->perf_event_enable &&
-		    try_module_get(event->mod)) {
-			ret = perf_trace_event_enable(event, data);
+	list_for_each_entry(tp_event, &ftrace_events, list) {
+		if (tp_event->id == event_id && tp_event->perf_event_enable &&
+		    try_module_get(tp_event->mod)) {
+			ret = perf_trace_event_init(tp_event, p_event);
 			break;
 		}
 	}
@@ -84,53 +100,52 @@ int perf_trace_enable(int event_id, void
 	return ret;
 }
 
-static void perf_trace_event_disable(struct ftrace_event_call *event)
+int perf_trace_enable(struct perf_event *p_event)
 {
-	if (--event->perf_refcount > 0)
-		return;
+	struct ftrace_event_call *tp_event = p_event->tp_event;
+	struct hlist_head *list;
 
-	event->perf_event_disable(event);
+	list = tp_event->perf_events;
+	if (WARN_ON_ONCE(!list))
+		return -EINVAL;
 

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113

Quoted 10 lines
kernel/trace/trace_event_perf.c +2 -8
-	if (!--total_ref_count) {
-		char *buf[4];
-		int i;
-
-		for (i = 0; i < 4; i++) {
-			buf[i] = perf_trace_buf[i];
-			rcu_assign_pointer(perf_trace_buf[i], NULL);
-		}
+	list = per_cpu_ptr(list, smp_processor_id());
+	hlist_add_head_rcu(&p_event->hlist_entry, list);

Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)

Quoted 23 lines
kernel/trace/trace_event_perf.c +8 -10
-		/*
-		 * Ensure every events in profiling have finished before
-		 * releasing the buffers
-		 */
-		synchronize_sched();
+	return 0;
+}
 
-		for (i = 0; i < 4; i++)
-			free_percpu(buf[i]);
-	}
+void perf_trace_disable(struct perf_event *p_event)
+{
+	hlist_del_rcu(&p_event->hlist_entry);
 }
 
-void perf_trace_disable(int event_id)
+void perf_trace_destroy(struct perf_event *p_event)
 {
-	struct ftrace_event_call *event;
+	struct ftrace_event_call *tp_event = p_event->tp_event;
+	int i;
 

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131

Quoted 10 lines
kernel/trace/trace_event_perf.c +4 -6
-	mutex_lock(&event_mutex);
-	list_for_each_entry(event, &ftrace_events, list) {
-		if (event->id == event_id) {
-			perf_trace_event_disable(event);
-			module_put(event->mod);
-			break;
+	if (--tp_event->perf_refcount > 0)
+		return;
+
+	tp_event->perf_event_disable(tp_event);

Don't we need a rcu_synchronize_sched() here?

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 132

Quoted 2 lines
kernel/trace/trace_event_perf.c +2 -0
+	free_percpu(tp_event->perf_events);
+	tp_event->perf_events = NULL;

And rcu_assign?

Quoted 22 lines
kernel/trace/trace_event_perf.c +6 -2
+
+	if (!--total_ref_count) {
+		for (i = 0; i < 4; i++) {
+			free_percpu(perf_trace_buf[i]);
+			perf_trace_buf[i] = NULL;
 		}
 	}
-	mutex_unlock(&event_mutex);
 }
 
 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
-	char *trace_buf, *raw_data;
+	char *raw_data;
 	int pc;
 
 	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
@@ -139,13 +154,9 @@ __kprobes void *perf_trace_buf_prepare(i
 
 	*rctxp = perf_swevent_get_recursion_context();

kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159

Quoted 10 lines
kernel/trace/trace_event_perf.c +2 -6
 	if (*rctxp < 0)
-		goto err_recursion;
-
-	trace_buf = rcu_dereference_sched(perf_trace_buf[*rctxp]);
-	if (!trace_buf)
-		goto err;
+		return NULL;
 
-	raw_data = per_cpu_ptr(trace_buf, smp_processor_id());
+	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());

Needs rcu_dereference_sched too. And this could be __this_cpu_var()

Quoted 26 lines
kernel/trace/trace_event_perf.c +0 -4
 	/* zero the dead bytes from align to not leak stack to user */
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
@@ -155,9 +166,5 @@ __kprobes void *perf_trace_buf_prepare(i
 	entry->type = type;
 
 	return raw_data;
-err:
-	perf_swevent_put_recursion_context(*rctxp);
-err_recursion:
-	return NULL;
 }
 EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);

Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================

kernel/trace/trace_kprobe.c +1 -0
@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(s
 	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;

kernel/trace/trace_kprobe.c static __kprobes void kprobe_perf_func(s · line 1365

Quoted 5 lines
kernel/trace/trace_kprobe.c +1 -1
@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());

Same

Quoted 13 lines
kernel/trace/trace_kprobe.c +2 -0
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
 /* Kretprobe profile handler */
@@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_fun
 	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kretprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;

kernel/trace/trace_kprobe.c static __kprobes void kretprobe_perf_fun · line 1398

Quoted 6 lines
kernel/trace/trace_kprobe.c +1 -2
@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_fun
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			      regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());

ditto

Quoted 17 lines
kernel/trace/trace_kprobe.c +1 -0
+	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)

Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================

kernel/trace/trace_syscalls.c +1 -0
@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;

kernel/trace/trace_syscalls.c static void perf_syscall_enter(struct pt · line 472

Quoted 7 lines
kernel/trace/trace_syscalls.c +2 -2
@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->enter_event->perf_data);
+
+	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());

ditto

Quoted 13 lines
kernel/trace/trace_syscalls.c +2 -0
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysenter_enable(struct ftrace_event_call *call)
@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;

kernel/trace/trace_syscalls.c static void perf_syscall_exit(struct pt_ · line 548

Quoted 6 lines
kernel/trace/trace_syscalls.c +1 -2
@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->exit_event->perf_data);
+	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());

and ditto :)

Quoted 59 lines
kernel/trace/trace_syscalls.c +1 -0
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysexit_enable(struct ftrace_event_call *call)

Index: linux-2.6/kernel/perf_event.c
===================================================================

kernel/perf_event.c +2 -12
@@ -4004,9 +4004,6 @@ static void perf_swevent_add(struct perf
 	perf_swevent_overflow(event, 0, nmi, data, regs);
 }
 
-static int perf_tp_event_match(struct perf_event *event,
-				struct perf_sample_data *data);
-
 static int perf_exclude_event(struct perf_event *event,
 			      struct pt_regs *regs)
 {
@@ -4036,10 +4033,6 @@ static int perf_swevent_match(struct per
 	if (perf_exclude_event(event, regs))
 		return 0;
 
-	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
-	    !perf_tp_event_match(event, data))
-		return 0;
-
 	return 1;
 }
 
@@ -4094,7 +4087,7 @@ end:
 
 int perf_swevent_get_recursion_context(void)
 {
-	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 	int rctx;
 
 	if (in_nmi())
@@ -4106,10 +4099,8 @@ int perf_swevent_get_recursion_context(v
 	else
 		rctx = 0;
 
-	if (cpuctx->recursion[rctx]) {
-		put_cpu_var(perf_cpu_context);
+	if (cpuctx->recursion[rctx])
 		return -1;
-	}
 
 	cpuctx->recursion[rctx]++;
 	barrier();
@@ -4123,7 +4114,6 @@ void perf_swevent_put_recursion_context(
 	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 	barrier();
 	cpuctx->recursion[rctx]--;
-	put_cpu_var(perf_cpu_context);
 }
 EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
 

kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127

Quoted 4 lines
kernel/perf_event.c +1 -0
@@ -4134,6 +4124,7 @@ void __perf_sw_event(u32 event_id, u64 n
 	struct perf_sample_data data;
 	int rctx;
 
+	preempt_disable_notrace();

Why is this needed. We have the recursion context protection already.

Ok, otherwise looks good.

#38 lore raw

Re: [PATCH 05/10] perf-record: Share per-cpu buffers

In reply to Peter Zijlstra
Quoted 39 lines · Peter Zijlstra

It seems a waste of space to create a buffer per event, share it per-cpu.

---
 tools/perf/builtin-record.c |   52 +++++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

Index: linux-2.6/tools/perf/builtin-record.c
===================================================================

tools/perf/builtin-record.c +4 -13
@@ -85,7 +85,7 @@ struct mmap_data {
 	unsigned int		prev;
 };
 
-static struct mmap_data		*mmap_array[MAX_NR_CPUS][MAX_COUNTERS];
+static struct mmap_data		mmap_array[MAX_NR_CPUS];
 
 static unsigned long mmap_read_head(struct mmap_data *md)
 {
@@ -380,18 +380,29 @@ try_again:
 		if (group && group_fd == -1)
 			group_fd = fd[nr_cpu][counter][thread_index];
 
-		event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
-		event_array[nr_poll].events = POLLIN;
-		nr_poll++;
-
-		mmap_array[nr_cpu][counter][thread_index].counter = counter;
-		mmap_array[nr_cpu][counter][thread_index].prev = 0;
-		mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
-		mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
-			PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
-		if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
-			error("failed to mmap with %d (%s)\n", errno, strerror(errno));
-			exit(-1);
+		if (counter || thread_index) {
+			ret = ioctl(fd[nr_cpu][counter][thread_index],
+					PERF_EVENT_IOC_SET_OUTPUT,

tools/perf/builtin-record.c try_again: · line 395

Quoted 10 lines
tools/perf/builtin-record.c +10 -0
+					fd[nr_cpu][0][0]);
+			if (ret) {
+				error("failed to set output: %d (%s)\n", errno,
+						strerror(errno));
+				exit(-1);
+			}
+		} else {
+			mmap_array[nr_cpu].counter = counter;
+			mmap_array[nr_cpu].prev = 0;
+			mmap_array[nr_cpu].mask = mmap_pages*page_size - 1;

If you do this multiplex per cpu (which I think is a very good thing), you should also increase the default value of mmap_pages as it might not be big enough anymore.

Quoted 46 lines
tools/perf/builtin-record.c +14 -11
+			mmap_array[nr_cpu].base = mmap(NULL, (mmap_pages+1)*page_size,
+				PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
+			if (mmap_array[nr_cpu].base == MAP_FAILED) {
+				error("failed to mmap with %d (%s)\n", errno, strerror(errno));
+				exit(-1);
+			}
+
+			event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
+			event_array[nr_poll].events = POLLIN;
+			nr_poll++;
 		}
 
 		if (filter != NULL) {
@@ -492,16 +503,11 @@ static struct perf_event_header finished
 
 static void mmap_read_all(void)
 {
-	int i, counter, thread;
+	int i;
 
 	for (i = 0; i < nr_cpu; i++) {
-		for (counter = 0; counter < nr_counters; counter++) {
-			for (thread = 0; thread < thread_num; thread++) {
-				if (mmap_array[i][counter][thread].base)
-					mmap_read(&mmap_array[i][counter][thread]);
-			}
-
-		}
+		if (mmap_array[i].base)
+			mmap_read(&mmap_array[i]);
 	}
 
 	if (perf_header__has_feat(&session->header, HEADER_TRACE_INFO))
@@ -876,9 +882,7 @@ int cmd_record(int argc, const char **ar
 	for (i = 0; i < MAX_NR_CPUS; i++) {
 		for (j = 0; j < MAX_COUNTERS; j++) {
 			fd[i][j] = malloc(sizeof(int)*thread_num);
-			mmap_array[i][j] = zalloc(
-				sizeof(struct mmap_data)*thread_num);
-			if (!fd[i][j] || !mmap_array[i][j])
+			if (!fd[i][j])
 				return -ENOMEM;
 		}
 	}
maintainer
#15 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Frederic Weisbecker
Quoted 9 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

									\

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794

Quoted 1 line
include/trace/ftrace.h +1 -0
+	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\

Should be rcu_dereference_sched ?

No, I removed all that rcu stuff and synchronized against the probe unregister.

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

Quoted 8 lines

kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44

Quoted 4 lines
kernel/trace/trace_event_perf.c +4 -0
+	for_each_possible_cpu(cpu)
+		INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
+
+	tp_event->perf_events = list;

I suspect this must be rcu_assign_pointer.

Same thing as above, I do this before probe register, so I see no need for RCU.

Quoted 7 lines

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113

Quoted 2 lines
kernel/trace/trace_event_perf.c +2 -0
+	list = per_cpu_ptr(list, smp_processor_id());
+	hlist_add_head_rcu(&p_event->hlist_entry, list);

Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)

What exactly is the puzzlement about?

Quoted 8 lines

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131

Quoted 4 lines
kernel/trace/trace_event_perf.c +4 -0
+	if (--tp_event->perf_refcount > 0)
+		return;
+
+	tp_event->perf_event_disable(tp_event);

Don't we need a rcu_synchronize_sched() here?

Doesn't probe unregister synchronize things against its own callback?

Quoted 6 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+	free_percpu(tp_event->perf_events);
+	tp_event->perf_events = NULL;

And rcu_assign?

Which again, makes any use of RCU unneeded.

Quoted 5 lines

kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159

Quoted 1 line
kernel/trace/trace_event_perf.c +1 -0
+	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());

Needs rcu_dereference_sched too. And this could be __this_cpu_var()

Ahh! so that is what its called.

Quoted 4 lines

kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127

Quoted 1 line
kernel/perf_event.c +1 -0
+	preempt_disable_notrace();

Why is this needed. We have the recursion context protection already.

Because:

@@ -4094,7 +4087,7 @@ end:

int perf_swevent_get_recursion_context(void)
{
- struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);

+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
        int rctx;
 
        if (in_nmi())
maintainer
#39 lore raw

Re: [PATCH 05/10] perf-record: Share per-cpu buffers

In reply to Frederic Weisbecker
Quoted 3 lines · Frederic Weisbecker

If you do this multiplex per cpu (which I think is a very good thing), you should also increase the default value of mmap_pages as it might not be big enough anymore.

Hmm, I didn't notice any problems with that.

#16 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

									\

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794

Quoted 1 line
include/trace/ftrace.h +1 -0
+	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\

Should be rcu_dereference_sched ?

Quoted 5 lines

No, I removed all that rcu stuff and synchronized against the probe unregister.

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.

There is a call_rcu that removes the callbacks, but that only protect the callback themselves.

Quoted 21 lines
Quoted 9 lines

kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44

Quoted 5 lines
kernel/trace/trace_event_perf.c +4 -0
+	for_each_possible_cpu(cpu)
+		INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
+
+	tp_event->perf_events = list;

I suspect this must be rcu_assign_pointer.

Same thing as above, I do this before probe register, so I see no need for RCU.

Quoted 7 lines

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113

Quoted 2 lines
kernel/trace/trace_event_perf.c +2 -0
+	list = per_cpu_ptr(list, smp_processor_id());
+	hlist_add_head_rcu(&p_event->hlist_entry, list);

Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)

Quoted 1 line

What exactly is the puzzlement about?

The fact we use the hlist API not for hlist purpose but for a list.

Quoted 10 lines
Quoted 8 lines

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131

Quoted 4 lines
kernel/trace/trace_event_perf.c +4 -0
+	if (--tp_event->perf_refcount > 0)
+		return;
+
+	tp_event->perf_event_disable(tp_event);

Don't we need a rcu_synchronize_sched() here?

Doesn't probe unregister synchronize things against its own callback?

May be I missed it but it doesn't seem so.

Quoted 7 lines
Quoted 5 lines

kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159

Quoted 1 line
kernel/trace/trace_event_perf.c +1 -0
+	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());

Needs rcu_dereference_sched too. And this could be __this_cpu_var()

Ahh! so that is what its called.

:)

Quoted 9 lines
Quoted 4 lines

kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127

Quoted 1 line
kernel/perf_event.c +1 -0
+	preempt_disable_notrace();

Why is this needed. We have the recursion context protection already.

Because:

kernel/perf_event.c end: · line 4093

Quoted 8 lines
kernel/perf_event.c +1 -1
@@ -4094,7 +4087,7 @@ end:
 
 int perf_swevent_get_recursion_context(void)
 {
-       struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+       struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
        int rctx;
 
        if (in_nmi())

Right.

Thanks.

maintainer
#17 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 3 lines · Frederic Weisbecker
Quoted 1 line

What exactly is the puzzlement about?

The fact we use the hlist API not for hlist purpose but for a list.

I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct list_head stuff.

#18 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
Quoted 1 line

What exactly is the puzzlement about?

The fact we use the hlist API not for hlist purpose but for a list.

I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct list_head stuff.

It's just that people might except this is used for a hlist purpose.

But nevermind, that's just a small issue.

maintainer
#21 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Frederic Weisbecker
Quoted 9 lines · Frederic Weisbecker
Quoted 2 lines

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.

There is a call_rcu that removes the callbacks, but that only protect the callback themselves.

Ah, ok, so we should do probe_unregister + synchronize_sched().

That should ensure __DO_TRACE() doesn't call into it anymore.

/me goes make a patch

#22 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 8 lines · Peter Zijlstra
Quoted 7 lines · Frederic Weisbecker
Quoted 2 lines

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.

Quoted 8 lines
Quoted 2 lines

There is a call_rcu that removes the callbacks, but that only protect the callback themselves.

Ah, ok, so we should do probe_unregister + synchronize_sched(). That should ensure __DO_TRACE() doesn't call into it anymore.

/me goes make a patch

Yep. But that also means we need to rcu_dereference_sched() to access the per cpu list of events.

Thanks.

maintainer
#23 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 15 lines · Frederic Weisbecker
Quoted 14 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 2 lines

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.

There is a call_rcu that removes the callbacks, but that only protect the callback themselves.

Ah, ok, so we should do probe_unregister + synchronize_sched(). That should ensure __DO_TRACE() doesn't call into it anymore.

Quoted 6 lines
Quoted 2 lines

/me goes make a patch

Yep. But that also means we need to rcu_dereference_sched() to access the per cpu list of events.

Why?

The per-cpu vars are allocated and freed in a fully serialized manner, there should be no races what so ever.

#24 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 20 lines · Peter Zijlstra
Quoted 19 lines · Frederic Weisbecker
Quoted 16 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 2 lines

I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.

I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.

There is a call_rcu that removes the callbacks, but that only protect the callback themselves.

Ah, ok, so we should do probe_unregister + synchronize_sched(). That should ensure __DO_TRACE() doesn't call into it anymore.

/me goes make a patch

Quoted 7 lines
Quoted 2 lines

Yep. But that also means we need to rcu_dereference_sched() to access the per cpu list of events.

Why?

The per-cpu vars are allocated and freed in a fully serialized manner, there should be no races what so ever.

Ah right, I was confused.

#19 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line

What exactly is the puzzlement about?

The fact we use the hlist API not for hlist purpose but for a list.

I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct list_head stuff.

Using an API in such a mixed way may cause puzzlement ;-)

Fortunately we've got the ultimate anti-puzzlement weapon:
code comments.

Ingo

maintainer
#25 lore raw

PATCH 1/1 perf, trace: Fix probe unregister race

In reply to Peter Zijlstra

tracepoint_probe_unregister() does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.

Also, use this_cpu_ptr() where possible.

---
 include/trace/ftrace.h          |    2 +-
 kernel/trace/trace_event_perf.c |   10 ++++++++--
 kernel/trace/trace_kprobe.c     |    4 ++--
 kernel/trace/trace_syscalls.c   |    4 ++--
 4 files changed, 13 insertions(+), 7 deletions(-)
include/trace/ftrace.h +1 -1
Index: linux-2.6/include/trace/ftrace.h
===================================================================
@@ -791,7 +791,7 @@ perf_trace_templ_##call(struct ftrace_ev
 									\
 	{ assign; }							\
 									\
-	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
+	head = this_cpu_ptr(event_call->perf_events);			\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
 		__count, __regs, head);					\
 }
kernel/trace/trace_event_perf.c +8 -2
Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================
@@ -109,7 +109,7 @@ int perf_trace_enable(struct perf_event 
 	if (WARN_ON_ONCE(!list))
 		return -EINVAL;
 
-	list = per_cpu_ptr(list, smp_processor_id());
+	list = this_cpu_ptr(list);
 	hlist_add_head_rcu(&p_event->hlist_entry, list);
 
 	return 0;
@@ -130,6 +130,12 @@ void perf_trace_destroy(struct perf_even
 
 	tp_event->perf_event_disable(tp_event);
 
+	/*
+	 * Ensure our callback won't be called anymore. See
+	 * tracepoint_probe_unregister() and __DO_TRACE().
+	 */
+	synchronize_sched();
+
 	free_percpu(tp_event->perf_events);
 	tp_event->perf_events = NULL;
 
@@ -156,7 +162,7 @@ __kprobes void *perf_trace_buf_prepare(i
 	if (*rctxp < 0)
 		return NULL;
 
-	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());
+	raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
 
 	/* zero the dead bytes from align to not leak stack to user */
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
kernel/trace/trace_kprobe.c +2 -2
Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================
@@ -1362,7 +1362,7 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
@@ -1395,7 +1395,7 @@ static __kprobes void kretprobe_perf_fun
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 
kernel/trace/trace_syscalls.c +2 -2
Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================
@@ -469,7 +469,7 @@ static void perf_syscall_enter(struct pt
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
 
-	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->enter_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
@@ -545,7 +545,7 @@ static void perf_syscall_exit(struct pt_
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->exit_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
Acked-by on v1 1/1
#26 lore raw

Re: [PATCH 02b/10] perf, trace: Fix probe unregister race

Quoted 7 lines · Peter Zijlstra

tracepoint_probe_unregister() does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.

Also, use this_cpu_ptr() where possible.

(and you previous patches too)

Quoted 101 lines
---
 include/trace/ftrace.h          |    2 +-
 kernel/trace/trace_event_perf.c |   10 ++++++++--
 kernel/trace/trace_kprobe.c     |    4 ++--
 kernel/trace/trace_syscalls.c   |    4 ++--
 4 files changed, 13 insertions(+), 7 deletions(-)

Index: linux-2.6/include/trace/ftrace.h
===================================================================

include/trace/ftrace.h +1 -1
@@ -791,7 +791,7 @@ perf_trace_templ_##call(struct ftrace_ev
 									\
 	{ assign; }							\
 									\
-	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
+	head = this_cpu_ptr(event_call->perf_events);			\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
 		__count, __regs, head);					\
 }

Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================

kernel/trace/trace_event_perf.c +8 -2
@@ -109,7 +109,7 @@ int perf_trace_enable(struct perf_event 
 	if (WARN_ON_ONCE(!list))
 		return -EINVAL;
 
-	list = per_cpu_ptr(list, smp_processor_id());
+	list = this_cpu_ptr(list);
 	hlist_add_head_rcu(&p_event->hlist_entry, list);
 
 	return 0;
@@ -130,6 +130,12 @@ void perf_trace_destroy(struct perf_even
 
 	tp_event->perf_event_disable(tp_event);
 
+	/*
+	 * Ensure our callback won't be called anymore. See
+	 * tracepoint_probe_unregister() and __DO_TRACE().
+	 */
+	synchronize_sched();
+
 	free_percpu(tp_event->perf_events);
 	tp_event->perf_events = NULL;
 
@@ -156,7 +162,7 @@ __kprobes void *perf_trace_buf_prepare(i
 	if (*rctxp < 0)
 		return NULL;
 
-	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());
+	raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
 
 	/* zero the dead bytes from align to not leak stack to user */
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));

Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================

kernel/trace/trace_kprobe.c +2 -2
@@ -1362,7 +1362,7 @@ static __kprobes void kprobe_perf_func(s
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
@@ -1395,7 +1395,7 @@ static __kprobes void kretprobe_perf_fun
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 

Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================

kernel/trace/trace_syscalls.c +2 -2
@@ -469,7 +469,7 @@ static void perf_syscall_enter(struct pt
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
 
-	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->enter_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
@@ -545,7 +545,7 @@ static void perf_syscall_exit(struct pt_
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->exit_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
#20 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Ingo Molnar
Quoted 9 lines · Ingo Molnar
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line

What exactly is the puzzlement about?

The fact we use the hlist API not for hlist purpose but for a list.

Quoted 6 lines
Quoted 3 lines

I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct list_head stuff.

Using an API in such a mixed way may cause puzzlement ;-)

I take that back - it's clear from the function prototype that we are using a mixed API: (hlist,list).

Sorry,

Ingo

maintainer
#48 lore raw

Re: [PATCH 09/10] perf: Remove more fastpath code

In reply to Peter Zijlstra

kernel/perf_event.c again: · line 2979

Quoted 12 lines · Peter Zijlstra

plain text document attachment (perf-opt-output-more.patch) Sanity checks cost instructions

---

kernel/perf_event.c +0 -7
-
-	/*
-	 * Check we didn't copy past our reservation window, taking the
-	 * possible unsigned int wrap into account.
-	 */
-	if (WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0))
-		return;

Are you sure you want to remove this? I mean, sanity checks are a small cost (I have several in the ftrace ring buffer) and can catch bugs in case something happens during development. Especially when the code is under a lot of flux.

I even have a sanity check that Ingo asked me to add, which would detect if a tracer (not the ring buffer, but the user of the ring buffer) recursed on itself. I think that check detected one bug in the function tracer in the begging, but hasn't caught anything since.

-- Steve

Quoted 27 lines
kernel/perf_event.c +3 -7
-
 	do {
-		unsigned long size = min(handle->size, len);
+		unsigned long size = min_t(unsigned long, handle->size, len);
 
 		memcpy(handle->addr, buf, size);
@@ -3054,15 +3045,12 @@ int perf_output_begin(struct perf_output
 			goto fail;
 	} while (local_cmpxchg(&data->head, offset, head) != offset);
 
-	handle->offset	= offset;
-	handle->head	= head;
-
 	if (head - local_read(&data->wakeup) > data->watermark)
 		local_add(data->watermark, &data->wakeup);
 
-	handle->page = handle->offset >> (PAGE_SHIFT + page_order(data));
+	handle->page = offset >> (PAGE_SHIFT + page_order(data));
 	handle->page &= data->nr_pages - 1;
-	handle->size = handle->offset & ((PAGE_SIZE << page_order(data)) - 1);
+	handle->size = offset & ((PAGE_SIZE << page_order(data)) - 1);
 	handle->addr = data->data_pages[handle->page];
 	handle->addr += handle->size;
 	handle->size = (PAGE_SIZE << page_order(data)) - handle->size;
maintainer
#49 lore raw

Re: [PATCH 09/10] perf: Remove more fastpath code

Quoted 7 lines · Steven Rostedt

kernel/perf_event.c again: · line 2979

Quoted 2 lines
kernel/perf_event.c +0 -2
-     if (WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0))
-             return;

Are you sure you want to remove this? I mean, sanity checks are a small cost (I have several in the ftrace ring buffer) and can catch bugs in case something happens during development. Especially when the code is under a lot of flux.

Yeah, I pondered adding a CONFIG_PERF_DEBUG for that.. thing is I was tracking 2 variables just for this, and that seemed a bit excessive.

Quoted 4 lines

I even have a sanity check that Ingo asked me to add, which would detect if a tracer (not the ring buffer, but the user of the ring buffer) recursed on itself. I think that check detected one bug in the function tracer in the begging, but hasn't caught anything since.

I still have recursion checks.

#28 lore raw

[tip:perf/core] perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra
Commit-ID:  1c024eca51fdc965290acf342ae16a476c2189d0
Gitweb:     http://git.kernel.org/tip/1c024eca51fdc965290acf342ae16a476c2189d0
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Wed, 19 May 2010 14:02:22 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:56 +0200

perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track events

Avoid the swevent hash-table by using per-tracepoint hlists.

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/ftrace_event.h    |   16 +++---
 include/linux/perf_event.h      |    6 +-
 include/trace/ftrace.h          |    4 +-
 kernel/perf_event.c             |   94 +++++++++++++++--------------
 kernel/trace/trace_event_perf.c |  127 ++++++++++++++++++++------------------
 kernel/trace/trace_kprobe.c     |    9 ++-
 kernel/trace/trace_syscalls.c   |   11 ++-
 7 files changed, 143 insertions(+), 124 deletions(-)
include/linux/ftrace_event.h +8 -8
index 126071b..7024b7d 100644
@@ -133,7 +133,7 @@ struct ftrace_event_call {
 	void			*data;
 
 	int			perf_refcount;
-	void			*perf_data;
+	struct hlist_head	*perf_events;
 	int			(*perf_event_enable)(struct ftrace_event_call *);
 	void			(*perf_event_disable)(struct ftrace_event_call *);
 };
@@ -192,9 +192,11 @@ struct perf_event;
 
 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
 
-extern int perf_trace_enable(int event_id, void *data);
-extern void perf_trace_disable(int event_id);
-extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
+extern int  perf_trace_init(struct perf_event *event);
+extern void perf_trace_destroy(struct perf_event *event);
+extern int  perf_trace_enable(struct perf_event *event);
+extern void perf_trace_disable(struct perf_event *event);
+extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,
 				     char *filter_str);
 extern void ftrace_profile_free_filter(struct perf_event *event);
 extern void *perf_trace_buf_prepare(int size, unsigned short type,
@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(int size, unsigned short type,
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-		       u64 count, struct pt_regs *regs, void *event)
+		       u64 count, struct pt_regs *regs, void *head)
 {
-	struct trace_entry *entry = raw_data;
-
-	perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
+	perf_tp_event(addr, count, raw_data, size, regs, head);
 	perf_swevent_put_recursion_context(rctx);
 }
 #endif
include/linux/perf_event.h +4 -2
index fe50347..7cd7b35 100644
@@ -727,6 +727,7 @@ struct perf_event {
 	perf_overflow_handler_t		overflow_handler;
 
 #ifdef CONFIG_EVENT_TRACING
+	struct ftrace_event_call	*tp_event;
 	struct event_filter		*filter;
 #endif
 
@@ -992,8 +993,9 @@ static inline bool perf_paranoid_kernel(void)
 }
 
 extern void perf_event_init(void);
-extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
-			  int entry_size, struct pt_regs *regs, void *event);
+extern void perf_tp_event(u64 addr, u64 count, void *record,
+			  int entry_size, struct pt_regs *regs,
+			  struct hlist_head *head);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags
include/trace/ftrace.h +3 -1
index f282885..4eb2148 100644
@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 	struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
 	struct ftrace_raw_##call *entry;				\
 	u64 __addr = 0, __count = 1;					\
+	struct hlist_head *head;					\
 	int __entry_size;						\
 	int __data_size;						\
 	int rctx;							\
@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call,		\
 									\
 	{ assign; }							\
 									\
+	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
-			       __count, __regs, event_call->perf_data);	\
+		__count, __regs, head);					\
 }
 
 #undef DEFINE_EVENT
kernel/perf_event.c +48 -46

173 lines not on this page · raw message

kernel/trace/trace_event_perf.c +67 -60

197 lines not on this page · raw message

kernel/trace/trace_kprobe.c +6 -3
index 20c96de..4681f60 100644
@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;
@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
 /* Kretprobe profile handler */
@@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 	struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
 	struct ftrace_event_call *call = &tp->call;
 	struct kretprobe_trace_entry_head *entry;
+	struct hlist_head *head;
 	u8 *data;
 	int size, __size, i;
 	int rctx;
@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
-			      regs, call->perf_data);
+	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 
 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +7 -4
index a657cef..eb769f2 100644
@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
 	rec->nr = syscall_nr;
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->enter_event->perf_data);
+
+	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysenter_enable(struct ftrace_event_call *call)
@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
+	struct hlist_head *head;
 	int syscall_nr;
 	int rctx;
 	int size;
@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
-			sys_data->exit_event->perf_data);
+	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
 int perf_sysexit_enable(struct ftrace_event_call *call)
#34 lore raw

[tip:perf/core] perf: Ensure that IOC_OUTPUT isn't used to create multi-writer buffers

In reply to Peter Zijlstra
Commit-ID:  0f139300c9057c16b5833a4636b715b104fe0baa
Gitweb:     http://git.kernel.org/tip/0f139300c9057c16b5833a4636b715b104fe0baa
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 14:35:15 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:57 +0200

perf: Ensure that IOC_OUTPUT isn't used to create multi-writer buffers

Since we want to ensure buffers only have a single writer, we must avoid creating one with multiple.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 kernel/perf_event.c |   19 +++++++++++++++++++
 1 files changed, 19 insertions(+), 0 deletions(-)
kernel/perf_event.c +19 -0
index 3f2cc31..7a93252 100644
@@ -4920,6 +4920,13 @@ static int perf_event_set_output(struct perf_event *event, int output_fd)
 	int fput_needed = 0;
 	int ret = -EINVAL;
 
+	/*
+	 * Don't allow output of inherited per-task events. This would
+	 * create performance issues due to cross cpu access.
+	 */
+	if (event->cpu == -1 && event->attr.inherit)
+		return -EINVAL;
+
 	if (!output_fd)
 		goto set;
 
@@ -4940,6 +4947,18 @@ static int perf_event_set_output(struct perf_event *event, int output_fd)
 	if (event->data)
 		goto out;
 
+	/*
+	 * Don't allow cross-cpu buffers
+	 */
+	if (output_event->cpu != event->cpu)
+		goto out;
+
+	/*
+	 * If its not a per-cpu buffer, it must be the same task.
+	 */
+	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
+		goto out;
+
 	atomic_long_inc(&output_file->f_count);
 
 set:
#36 lore raw

[tip:perf/core] perf-record: Remove -M

In reply to Peter Zijlstra
Commit-ID:  57adc51dce9102b6641269dd04f5b99aac83b820
Gitweb:     http://git.kernel.org/tip/57adc51dce9102b6641269dd04f5b99aac83b820
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 14:39:55 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:57 +0200

perf-record: Remove -M

Since it is not allowed to create cross-cpu (or cross-task) buffers, this option is no longer valid.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 tools/perf/builtin-record.c |   36 ++++++++++++------------------------
 1 files changed, 12 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +12 -24
index e672269..94e210f 100644
@@ -61,8 +61,6 @@ static bool			call_graph			=  false;
 static bool			inherit_stat			=  false;
 static bool			no_samples			=  false;
 static bool			sample_address			=  false;
-static bool			multiplex			=  false;
-static int			multiplex_fd			=     -1;
 
 static long			samples				=      0;
 static u64			bytes_written			=      0;
@@ -366,27 +364,19 @@ try_again:
 		 */
 		if (group && group_fd == -1)
 			group_fd = fd[nr_cpu][counter][thread_index];
-		if (multiplex && multiplex_fd == -1)
-			multiplex_fd = fd[nr_cpu][counter][thread_index];
 
-		if (multiplex && fd[nr_cpu][counter][thread_index] != multiplex_fd) {
-
-			ret = ioctl(fd[nr_cpu][counter][thread_index], PERF_EVENT_IOC_SET_OUTPUT, multiplex_fd);
-			assert(ret != -1);
-		} else {
-			event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
-			event_array[nr_poll].events = POLLIN;
-			nr_poll++;
-
-			mmap_array[nr_cpu][counter][thread_index].counter = counter;
-			mmap_array[nr_cpu][counter][thread_index].prev = 0;
-			mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
-			mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
-				PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
-			if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
-				error("failed to mmap with %d (%s)\n", errno, strerror(errno));
-				exit(-1);
-			}
+		event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
+		event_array[nr_poll].events = POLLIN;
+		nr_poll++;
+
+		mmap_array[nr_cpu][counter][thread_index].counter = counter;
+		mmap_array[nr_cpu][counter][thread_index].prev = 0;
+		mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
+		mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
+			PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
+		if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
+			error("failed to mmap with %d (%s)\n", errno, strerror(errno));
+			exit(-1);
 		}
 
 		if (filter != NULL) {
@@ -820,8 +810,6 @@ static const struct option options[] = {
 		    "Sample addresses"),
 	OPT_BOOLEAN('n', "no-samples", &no_samples,
 		    "don't sample"),
-	OPT_BOOLEAN('M', "multiplex", &multiplex,
-		    "multiplex counter output in a single channel"),
 	OPT_END()
 };
 
#40 lore raw

[tip:perf/core] perf-record: Share per-cpu buffers

In reply to Peter Zijlstra
Commit-ID:  0e2e63dd608bf5844ffae7bf7d860de18a62724c
Gitweb:     http://git.kernel.org/tip/0e2e63dd608bf5844ffae7bf7d860de18a62724c
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 14:45:26 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:58 +0200

perf-record: Share per-cpu buffers

It seems a waste of space to create a buffer per event, share it per-cpu.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 tools/perf/builtin-record.c |   52 +++++++++++++++++++++++-------------------
 1 files changed, 28 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +28 -24
index 94e210f..9bc8905 100644
@@ -82,7 +82,7 @@ struct mmap_data {
 	unsigned int		prev;
 };
 
-static struct mmap_data		*mmap_array[MAX_NR_CPUS][MAX_COUNTERS];
+static struct mmap_data		mmap_array[MAX_NR_CPUS];
 
 static unsigned long mmap_read_head(struct mmap_data *md)
 {
@@ -365,18 +365,29 @@ try_again:
 		if (group && group_fd == -1)
 			group_fd = fd[nr_cpu][counter][thread_index];
 
-		event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
-		event_array[nr_poll].events = POLLIN;
-		nr_poll++;
-
-		mmap_array[nr_cpu][counter][thread_index].counter = counter;
-		mmap_array[nr_cpu][counter][thread_index].prev = 0;
-		mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size - 1;
-		mmap_array[nr_cpu][counter][thread_index].base = mmap(NULL, (mmap_pages+1)*page_size,
-			PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
-		if (mmap_array[nr_cpu][counter][thread_index].base == MAP_FAILED) {
-			error("failed to mmap with %d (%s)\n", errno, strerror(errno));
-			exit(-1);
+		if (counter || thread_index) {
+			ret = ioctl(fd[nr_cpu][counter][thread_index],
+					PERF_EVENT_IOC_SET_OUTPUT,
+					fd[nr_cpu][0][0]);
+			if (ret) {
+				error("failed to set output: %d (%s)\n", errno,
+						strerror(errno));
+				exit(-1);
+			}
+		} else {
+			mmap_array[nr_cpu].counter = counter;
+			mmap_array[nr_cpu].prev = 0;
+			mmap_array[nr_cpu].mask = mmap_pages*page_size - 1;
+			mmap_array[nr_cpu].base = mmap(NULL, (mmap_pages+1)*page_size,
+				PROT_READ|PROT_WRITE, MAP_SHARED, fd[nr_cpu][counter][thread_index], 0);
+			if (mmap_array[nr_cpu].base == MAP_FAILED) {
+				error("failed to mmap with %d (%s)\n", errno, strerror(errno));
+				exit(-1);
+			}
+
+			event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index];
+			event_array[nr_poll].events = POLLIN;
+			nr_poll++;
 		}
 
 		if (filter != NULL) {
@@ -477,16 +488,11 @@ static struct perf_event_header finished_round_event = {
 
 static void mmap_read_all(void)
 {
-	int i, counter, thread;
+	int i;
 
 	for (i = 0; i < nr_cpu; i++) {
-		for (counter = 0; counter < nr_counters; counter++) {
-			for (thread = 0; thread < thread_num; thread++) {
-				if (mmap_array[i][counter][thread].base)
-					mmap_read(&mmap_array[i][counter][thread]);
-			}
-
-		}
+		if (mmap_array[i].base)
+			mmap_read(&mmap_array[i]);
 	}
 
 	if (perf_header__has_feat(&session->header, HEADER_TRACE_INFO))
@@ -861,9 +867,7 @@ int cmd_record(int argc, const char **argv, const char *prefix __used)
 	for (i = 0; i < MAX_NR_CPUS; i++) {
 		for (j = 0; j < MAX_COUNTERS; j++) {
 			fd[i][j] = malloc(sizeof(int)*thread_num);
-			mmap_array[i][j] = zalloc(
-				sizeof(struct mmap_data)*thread_num);
-			if (!fd[i][j] || !mmap_array[i][j])
+			if (!fd[i][j])
 				return -ENOMEM;
 		}
 	}
#42 lore raw

[tip:perf/core] perf: Fix wakeup storm for RO mmap()s

In reply to Peter Zijlstra
Commit-ID:  adb8e118f288dc4c569ac9a89010b81a4745fbf0
Gitweb:     http://git.kernel.org/tip/adb8e118f288dc4c569ac9a89010b81a4745fbf0
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 16:21:55 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:58 +0200

perf: Fix wakeup storm for RO mmap()s

RO mmap()s don't update the tail pointer, so comparing against it for determining the written data size doesn't really do any good.

Keep track of when we last did a wakeup, and compare against that.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |    2 +-
 kernel/perf_event.c        |    4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
include/linux/perf_event.h +1 -1
index 7cd7b35..7098ebb 100644
@@ -599,7 +599,7 @@ struct perf_mmap_data {
 	local_t				head;		/* write position    */
 	local_t				nest;		/* nested writers    */
 	local_t				events;		/* event limit       */
-	local_t				wakeup;		/* needs a wakeup    */
+	local_t				wakeup;		/* wakeup stamp      */
 	local_t				lost;		/* nr records lost   */
 
 	long				watermark;	/* wakeup watermark  */
kernel/perf_event.c +2 -2
index 7a93252..1531e0b 100644
@@ -3056,8 +3056,8 @@ int perf_output_begin(struct perf_output_handle *handle,
 	handle->offset	= offset;
 	handle->head	= head;
 
-	if (head - tail > data->watermark)
-		local_inc(&data->wakeup);
+	if (head - local_read(&data->wakeup) > data->watermark)
+		local_add(data->watermark, &data->wakeup);
 
 	if (have_lost) {
 		lost_event.header.type = PERF_RECORD_LOST;
#44 lore raw

[tip:perf/core] perf: Optimize perf_output_copy()

In reply to Peter Zijlstra
Commit-ID:  5d967a8be636a4f301a8daad642bd1007299d9ec
Gitweb:     http://git.kernel.org/tip/5d967a8be636a4f301a8daad642bd1007299d9ec
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 16:46:39 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:59 +0200

perf: Optimize perf_output_copy()

Reduce the clutter in perf_output_copy() by keeping an interator in perf_output_handle.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |    3 ++
 kernel/perf_event.c        |   54 +++++++++++++++++++++----------------------
 2 files changed, 29 insertions(+), 28 deletions(-)
include/linux/perf_event.h +3 -0
index 7098ebb..7bd17f0 100644
@@ -806,6 +806,9 @@ struct perf_output_handle {
 	unsigned long			head;
 	unsigned long			offset;
 	unsigned long			wakeup;
+	unsigned long			size;
+	void				*addr;
+	int				page;
 	int				nmi;
 	int				sample;
 };
kernel/perf_event.c +26 -28

71 lines not on this page · raw message

#46 lore raw

[tip:perf/core] perf: Optimize the !vmalloc backed buffer

In reply to Peter Zijlstra
Commit-ID:  3cafa9fbb5c1d564b7b8e7224f493effbf04ffee
Gitweb:     http://git.kernel.org/tip/3cafa9fbb5c1d564b7b8e7224f493effbf04ffee
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 19:07:56 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:59 +0200

perf: Optimize the !vmalloc backed buffer

Reduce code and data by using the knowledge that for !PERF_USE_VMALLOC data_order is always 0.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |    2 +-
 kernel/perf_event.c        |   41 ++++++++++++++++++++++++++---------------
 2 files changed, 27 insertions(+), 16 deletions(-)
include/linux/perf_event.h +1 -1
index 7bd17f0..09cd9c1 100644
@@ -588,8 +588,8 @@ struct perf_mmap_data {
 	struct rcu_head			rcu_head;
 #ifdef CONFIG_PERF_USE_VMALLOC
 	struct work_struct		work;
+	int				page_order;	/* allocation order  */
 #endif
-	int				data_order;	/* allocation order  */
 	int				nr_pages;	/* nr of data pages  */
 	int				writable;	/* are we writable   */
 	int				nr_locked;	/* nr pages mlocked  */
kernel/perf_event.c +26 -15

102 lines not on this page · raw message

#50 lore raw

[tip:perf/core] perf: Remove more code from the fastpath

In reply to Peter Zijlstra
Commit-ID:  a94ffaaf55552769af328eaca9260fe6291c66c7
Gitweb:     http://git.kernel.org/tip/a94ffaaf55552769af328eaca9260fe6291c66c7
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 19:50:07 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:37:59 +0200

perf: Remove more code from the fastpath

Sanity checks cost instructions.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |    2 --
 kernel/perf_event.c        |   20 ++++----------------
 2 files changed, 4 insertions(+), 18 deletions(-)
include/linux/perf_event.h +0 -2
index 09cd9c1..fb6c91e 100644
@@ -803,8 +803,6 @@ struct perf_cpu_context {
 struct perf_output_handle {
 	struct perf_event		*event;
 	struct perf_mmap_data		*data;
-	unsigned long			head;
-	unsigned long			offset;
 	unsigned long			wakeup;
 	unsigned long			size;
 	void				*addr;
kernel/perf_event.c +4 -16
index 953ce46..d25c864 100644
@@ -2967,20 +2967,11 @@ again:
 	preempt_enable();
 }
 
-void perf_output_copy(struct perf_output_handle *handle,
+__always_inline void perf_output_copy(struct perf_output_handle *handle,
 		      const void *buf, unsigned int len)
 {
-	handle->offset += len;
-
-	/*
-	 * Check we didn't copy past our reservation window, taking the
-	 * possible unsigned int wrap into account.
-	 */
-	if (WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0))
-		return;
-
 	do {
-		unsigned long size = min(handle->size, len);
+		unsigned long size = min_t(unsigned long, handle->size, len);
 
 		memcpy(handle->addr, buf, size);
 
@@ -3055,15 +3046,12 @@ int perf_output_begin(struct perf_output_handle *handle,
 			goto fail;
 	} while (local_cmpxchg(&data->head, offset, head) != offset);
 
-	handle->offset	= offset;
-	handle->head	= head;
-
 	if (head - local_read(&data->wakeup) > data->watermark)
 		local_add(data->watermark, &data->wakeup);
 
-	handle->page = handle->offset >> (PAGE_SHIFT + page_order(data));
+	handle->page = offset >> (PAGE_SHIFT + page_order(data));
 	handle->page &= data->nr_pages - 1;
-	handle->size = handle->offset & ((PAGE_SIZE << page_order(data)) - 1);
+	handle->size = offset & ((PAGE_SIZE << page_order(data)) - 1);
 	handle->addr = data->data_pages[handle->page];
 	handle->addr += handle->size;
 	handle->size = (PAGE_SIZE << page_order(data)) - handle->size;
#52 lore raw

[tip:perf/core] perf: Optimize perf_tp_event_match()

In reply to Peter Zijlstra
Commit-ID:  580d607cd666dfabfc1c7b0fb08c8ac690c7c87f
Gitweb:     http://git.kernel.org/tip/580d607cd666dfabfc1c7b0fb08c8ac690c7c87f
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 May 2010 20:54:31 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 21 May 2010 11:38:00 +0200

perf: Optimize perf_tp_event_match()

Since we know tracepoints come from kernel context, avoid conditionals that try and establish that very fact.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <[email protected]>

---
 kernel/perf_event.c |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)
kernel/perf_event.c +4 -1
index d25c864..e099650 100644
@@ -4496,7 +4496,10 @@ static int perf_tp_event_match(struct perf_event *event,
 				struct perf_sample_data *data,
 				struct pt_regs *regs)
 {
-	if (perf_exclude_event(event, regs))
+	/*
+	 * All tracepoints are from kernel-space.
+	 */
+	if (event->attr.exclude_kernel)
 		return 0;
 
 	if (!perf_tp_filter_match(event, data))
maintainer
#29 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

In reply to Peter Zijlstra
Quoted 13 lines · Peter Zijlstra

plain text document attachment (perf-trace-per-cpu-fold.patch) Avoid the swevent hash-table by using per-tracepoint hlists.

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

kernel/trace/trace_event_perf.c +3 -2
-void perf_trace_disable(int event_id)
+void perf_trace_destroy(struct perf_event *p_event)
 {
-	struct ftrace_event_call *event;
+	struct ftrace_event_call *tp_event = p_event->tp_event;
+	int i;

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 130

Quoted 10 lines
kernel/trace/trace_event_perf.c +3 -6
 
-	mutex_lock(&event_mutex);
-	list_for_each_entry(event, &ftrace_events, list) {
-		if (event->id == event_id) {
-			perf_trace_event_disable(event);
-			module_put(event->mod);
-			break;
+	if (--tp_event->perf_refcount > 0)
+		return;
+

You remove the event_mutex and then update the tp_event->perf_refcount without any protection.

That tp_event->perf_refcount is global to the event not to the perf event.

-- Steve

Quoted 14 lines
kernel/trace/trace_event_perf.c +9 -1
+	tp_event->perf_event_disable(tp_event);
+
+	free_percpu(tp_event->perf_events);
+	tp_event->perf_events = NULL;
+
+	if (!--total_ref_count) {
+		for (i = 0; i < 4; i++) {
+			free_percpu(perf_trace_buf[i]);
+			perf_trace_buf[i] = NULL;
 		}
 	}
-	mutex_unlock(&event_mutex);
 }
maintainer
#30 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 28 lines · Steven Rostedt
Quoted 13 lines · Peter Zijlstra

plain text document attachment (perf-trace-per-cpu-fold.patch) Avoid the swevent hash-table by using per-tracepoint hlists.

Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.

kernel/trace/trace_event_perf.c +3 -2
-void perf_trace_disable(int event_id)
+void perf_trace_destroy(struct perf_event *p_event)
 {
-	struct ftrace_event_call *event;
+	struct ftrace_event_call *tp_event = p_event->tp_event;
+	int i;

kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 130

Quoted 10 lines
kernel/trace/trace_event_perf.c +3 -6
 
-	mutex_lock(&event_mutex);
-	list_for_each_entry(event, &ftrace_events, list) {
-		if (event->id == event_id) {
-			perf_trace_event_disable(event);
-			module_put(event->mod);
-			break;
+	if (--tp_event->perf_refcount > 0)
+		return;
+

You remove the event_mutex and then update the tp_event->perf_refcount without any protection.

Quoted 2 lines

That tp_event->perf_refcount is global to the event not to the perf event.

Oh, bugger, yes. I seem to have lost it on destroy..

Thanks!

maintainer
#31 lore raw

Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events

Quoted 5 lines · Peter Zijlstra
Quoted 3 lines

That tp_event->perf_refcount is global to the event not to the perf event.

Oh, bugger, yes. I seem to have lost it on destroy..

---
Subject: perf, trace: Fix perf_trace_destroy(), mutex went missing
From: Peter Zijlstra <[email protected]>
Date: Fri May 21 16:22:33 CEST 2010

Steve spotted I forgot to do the destroy under event_mutex.

LKML-Reference: <new-submission>

---
 kernel/trace/trace_event_perf.c |    5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
kernel/trace/trace_event_perf.c +4 -1
Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================
@@ -125,8 +125,9 @@ void perf_trace_destroy(struct perf_even
 	struct ftrace_event_call *tp_event = p_event->tp_event;
 	int i;
 
+	mutex_lock(&event_mutex);
 	if (--tp_event->perf_refcount > 0)
-		return;
+		goto out;
 
 	tp_event->perf_event_disable(tp_event);
 
@@ -145,6 +146,8 @@ void perf_trace_destroy(struct perf_even
 			perf_trace_buf[i] = NULL;
 		}
 	}
+out:
+	mutex_unlock(&event_mutex);
 }
 
 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
#2 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

In reply to Peter Zijlstra
Quoted 7 lines · Peter Zijlstra

[...]

include/trace/ftrace.h +0 -2
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev
 static notrace void perf_trace_##call(proto)				\
 {									\
 	struct ftrace_event_call *event_call = &event_##call;		\
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-									\

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 806

Quoted 10 lines
include/trace/ftrace.h +3 -4
-	perf_fetch_caller_regs(__regs, 1);				\
-									\
-	perf_trace_templ_##template(event_call, __regs, args);		\
+	struct pt_regs __regs;						\
 									\
-	put_cpu_var(perf_trace_regs);					\
+	perf_fetch_caller_regs(&__regs, 1);				\
+	perf_trace_templ_##template(event_call, &__regs, args);		\
 }
[...]

To what extent are you worried about something the size of struct pt_regs being auto/stack allocated?

- FChE

maintainer
#3 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

Quoted 14 lines · Frank Ch. Eigler
Quoted 12 lines · Peter Zijlstra

[...]

include/trace/ftrace.h +1 -5
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev
 static notrace void perf_trace_##call(proto)				\
 {									\
 	struct ftrace_event_call *event_call = &event_##call;		\
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-									\
-	perf_fetch_caller_regs(__regs, 1);				\
-									\
-	perf_trace_templ_##template(event_call, __regs, args);		\
+	struct pt_regs __regs;						\
 									\
Quoted 8 lines

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 806

Quoted 5 lines
include/trace/ftrace.h +2 -1
-	put_cpu_var(perf_trace_regs);					\
+	perf_fetch_caller_regs(&__regs, 1);				\
+	perf_trace_templ_##template(event_call, &__regs, args);		\
 }
[...]

To what extent are you worried about something the size of struct pt_regs being auto/stack allocated?

Isn't pt_regs already allocated on the stack when interrupted?

-- Steve

#4 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

Hi -

Quoted 10 lines · Steven Rostedt

[...]

Quoted 7 lines

include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 802

Quoted 4 lines
include/trace/ftrace.h +1 -2
-	struct pt_regs *__regs = &get_cpu_var(perf_trace_regs);		\
-	[...]
+	struct pt_regs __regs;						\
[...]

To what extent are you worried about something the size of struct pt_regs being auto/stack allocated?

Isn't pt_regs already allocated on the stack when interrupted?

Maybe, but here we're about to ask for another multi-hundred-byte one. Does your stack-frame-size ftrace widget gripe about this change?

- FChE

#5 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

In reply to Peter Zijlstra

kernel/trace/trace_event_perf.c void perf_trace_disable(int event_id) · line 155

Quoted 4 lines · Peter Zijlstra
kernel/trace/trace_event_perf.c +1 -1
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, *irq_flags, pc);
+	tracing_generic_entry_update(entry, regs->flags, pc);
 	entry->type = type;

This is breaking on powerpc -- we don't have a "flags" element in struct pt_regs. What is it trying to do? Get the interrupt enable state, as local_irq_save(flags) would provide?

Paul.

maintainer
#6 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

Quoted 10 lines · Paul Mackerras

Review

Quoted 4 lines · Peter Zijlstra
(quoted diff) +1 -1
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, *irq_flags, pc);
+	tracing_generic_entry_update(entry, regs->flags, pc);
 	entry->type = type;

This is breaking on powerpc -- we don't have a "flags" element in struct pt_regs. What is it trying to do? Get the interrupt enable state, as local_irq_save(flags) would provide?

Yes, I fouled that up.

On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().

So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.

Possibly we could remove the local_save_flags() from the perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags().

Something like:

---
 arch/x86/kernel/cpu/perf_event.c |    1 -
 kernel/trace/trace_event_perf.c  |    4 +++-
 2 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
index fd4db0d..856aeeb 100644
@@ -1717,7 +1717,6 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski
 	 */
 	regs->bp = rewind_frame_pointer(skip + 1);
 	regs->cs = __KERNEL_CS;
-	local_save_flags(regs->flags);
 }
 
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
index 26b8607..cb6f365 100644
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
+	unsigned long flags;
 	char *raw_data;
 	int pc;
 
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, regs->flags, pc);
+	local_save_flags(flags);
+	tracing_generic_entry_update(entry, flags, pc);
 	entry->type = type;
 
 	return raw_data;
#7 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

Quoted 13 lines · Peter Zijlstra
Quoted 3 lines · Paul Mackerras

This is breaking on powerpc -- we don't have a "flags" element in struct pt_regs. What is it trying to do? Get the interrupt enable state, as local_irq_save(flags) would provide?

Yes, I fouled that up.

On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().

So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.

Quoted 6 lines

Possibly we could remove the local_save_flags() from the perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags().

Something like:

Yes, that works -- at least, it compiles and seems to run. :)

Paul.

#11 lore raw

Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction

In reply to Peter Zijlstra
Quoted 17 lines · Peter Zijlstra
Quoted 10 lines · Paul Mackerras

kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, · line 180

Quoted 4 lines · Peter Zijlstra
kernel/trace/trace_event_perf.c +1 -1
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, *irq_flags, pc);
+	tracing_generic_entry_update(entry, regs->flags, pc);
 	entry->type = type;

This is breaking on powerpc -- we don't have a "flags" element in struct pt_regs. What is it trying to do? Get the interrupt enable state, as local_irq_save(flags) would provide?

Yes, I fouled that up.

On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().

Quoted 7 lines

So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.

Possibly we could remove the local_save_flags() from the perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags().

It was an anticipation for further uses. But since further are not appearing, you can safely remove it.

Thanks.

maintainer
#12 lore raw

PATCH 1/1 perf, trace: Fix !x86 build issue

In reply to Peter Zijlstra

Subject: perf, trace: Fix !x86 build issue
From: Peter Zijlstra <[email protected]>
Date: Tue May 25 09:22:38 CEST 2010

Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.

The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.

Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.

---
 arch/x86/kernel/cpu/perf_event.c |    6 +++++-
 kernel/trace/trace_event_perf.c  |    4 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct 
 	 */
 	regs->bp = rewind_frame_pointer(skip + 1);
 	regs->cs = __KERNEL_CS;
-	local_save_flags(regs->flags);
+	/*
+	 * We abuse bit 3 to pass exact information, see perf_misc_flags
+	 * and the comment with PERF_EFLAGS_EXACT.
+	 */
+	regs->flags = 0;
 }
 
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(i
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
+	unsigned long flags;
 	char *raw_data;
 	int pc;
 
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(i
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, regs->flags, pc);
+	local_save_flags(flags);
+	tracing_generic_entry_update(entry, flags, pc);
 	entry->type = type;
 
 	return raw_data;
#8 lore raw

[tip:perf/core] perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction

In reply to Paul Mackerras
Commit-ID:  23ef672d1e8236e6fe606a39963ba2d576972d84
Gitweb:     http://git.kernel.org/tip/23ef672d1e8236e6fe606a39963ba2d576972d84
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Sun, 23 May 2010 20:16:27 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 25 May 2010 09:02:08 +0200

perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction

On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().

So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.

[ Possibly we could remove the local_save_flags() from the
perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags(). ]

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <20100524042958.GA10628@drongo>

---
 arch/x86/kernel/cpu/perf_event.c |    1 -
 kernel/trace/trace_event_perf.c  |    4 +++-
 2 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
index fd4db0d..856aeeb 100644
@@ -1717,7 +1717,6 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski
 	 */
 	regs->bp = rewind_frame_pointer(skip + 1);
 	regs->cs = __KERNEL_CS;
-	local_save_flags(regs->flags);
 }
 
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
index 26b8607..cb6f365 100644
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
+	unsigned long flags;
 	char *raw_data;
 	int pc;
 
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, regs->flags, pc);
+	local_save_flags(flags);
+	tracing_generic_entry_update(entry, flags, pc);
 	entry->type = type;
 
 	return raw_data;
maintainer Acked-by Paul Mackerras
#9 lore raw

Re: [tip:perf/core] perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction

Quoted 20 lines · tip-bot for Peter Zijlstra
Commit-ID:  23ef672d1e8236e6fe606a39963ba2d576972d84
Gitweb:     http://git.kernel.org/tip/23ef672d1e8236e6fe606a39963ba2d576972d84
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Sun, 23 May 2010 20:16:27 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 25 May 2010 09:02:08 +0200

perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction

On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().

So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.

[ Possibly we could remove the local_save_flags() from the
perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags(). ]

Quoted 8 lines
Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Frederic Weisbecker <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Steven Rostedt <[email protected]>

LKML-Reference: <20100524042958.GA10628@drongo>

Could you replace this with:

---
Subject: perf, trace: Fix !x86 build issue
From: Peter Zijlstra <[email protected]>
Date: Tue May 25 09:22:38 CEST 2010

Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.

The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.

Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.

---
 arch/x86/kernel/cpu/perf_event.c |    6 +++++-
 kernel/trace/trace_event_perf.c  |    4 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct 
 	 */
 	regs->bp = rewind_frame_pointer(skip + 1);
 	regs->cs = __KERNEL_CS;
-	local_save_flags(regs->flags);
+	/*
+	 * We abuse bit 3 to pass exact information, see perf_misc_flags
+	 * and the comment with PERF_EFLAGS_EXACT.
+	 */
+	regs->flags = 0;
 }
 
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(i
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
+	unsigned long flags;
 	char *raw_data;
 	int pc;
 
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(i
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, regs->flags, pc);
+	local_save_flags(flags);
+	tracing_generic_entry_update(entry, flags, pc);
 	entry->type = type;
 
 	return raw_data;
#10 lore raw

[tip:perf/core] perf, trace: Fix !x86 build bug

Commit-ID:  87f44bbc246c5244c76a701f8eefba7788bce64a
Gitweb:     http://git.kernel.org/tip/87f44bbc246c5244c76a701f8eefba7788bce64a
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Tue, 25 May 2010 11:02:55 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 25 May 2010 11:28:49 +0200

perf, trace: Fix !x86 build bug

Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.

The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.

Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.

LKML-Reference: <1274778175.5882.623.camel@twins>

---
 arch/x86/kernel/cpu/perf_event.c |    6 +++++-
 kernel/trace/trace_event_perf.c  |    4 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
index fd4db0d..c775860 100644
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski
 	 */
 	regs->bp = rewind_frame_pointer(skip + 1);
 	regs->cs = __KERNEL_CS;
-	local_save_flags(regs->flags);
+	/*
+	 * We abuse bit 3 to pass exact information, see perf_misc_flags
+	 * and the comment with PERF_EFLAGS_EXACT.
+	 */
+	regs->flags = 0;
 }
 
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
index 26b8607..cb6f365 100644
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 				       struct pt_regs *regs, int *rctxp)
 {
 	struct trace_entry *entry;
+	unsigned long flags;
 	char *raw_data;
 	int pc;
 
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
 
 	entry = (struct trace_entry *)raw_data;
-	tracing_generic_entry_update(entry, regs->flags, pc);
+	local_save_flags(flags);
+	tracing_generic_entry_update(entry, flags, pc);
 	entry->type = type;
 
 	return raw_data;
#27 lore raw

[tip:perf/urgent] perf_events, trace: Fix probe unregister race

In reply to Peter Zijlstra
Commit-ID:  3771f0771154675d4a0ca780be2411f3cc357208
Gitweb:     http://git.kernel.org/tip/3771f0771154675d4a0ca780be2411f3cc357208
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Fri, 21 May 2010 12:31:09 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 31 May 2010 08:46:09 +0200

perf_events, trace: Fix probe unregister race

tracepoint_probe_unregister() does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.

Also, use this_cpu_ptr() where possible.

LKML-Reference: <1274438476.1674.1702.camel@laptop>

---
 include/trace/ftrace.h          |    2 +-
 kernel/trace/trace_event_perf.c |   10 ++++++++--
 kernel/trace/trace_kprobe.c     |    4 ++--
 kernel/trace/trace_syscalls.c   |    4 ++--
 4 files changed, 13 insertions(+), 7 deletions(-)
include/trace/ftrace.h +1 -1
index 3d685d1..5a64905 100644
@@ -725,7 +725,7 @@ perf_trace_##call(void *__data, proto)					\
 									\
 	{ assign; }							\
 									\
-	head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
+	head = this_cpu_ptr(event_call->perf_events);			\
 	perf_trace_buf_submit(entry, __entry_size, rctx, __addr,	\
 		__count, &__regs, head);				\
 }
kernel/trace/trace_event_perf.c +8 -2
index cb6f365..49c7abf 100644
@@ -116,7 +116,7 @@ int perf_trace_enable(struct perf_event *p_event)
 	if (WARN_ON_ONCE(!list))
 		return -EINVAL;
 
-	list = per_cpu_ptr(list, smp_processor_id());
+	list = this_cpu_ptr(list);
 	hlist_add_head_rcu(&p_event->hlist_entry, list);
 
 	return 0;
@@ -142,6 +142,12 @@ void perf_trace_destroy(struct perf_event *p_event)
 					    tp_event->class->perf_probe,
 					    tp_event);
 
+	/*
+	 * Ensure our callback won't be called anymore. See
+	 * tracepoint_probe_unregister() and __DO_TRACE().
+	 */
+	synchronize_sched();
+
 	free_percpu(tp_event->perf_events);
 	tp_event->perf_events = NULL;
 
@@ -169,7 +175,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
 	if (*rctxp < 0)
 		return NULL;
 
-	raw_data = per_cpu_ptr(perf_trace_buf[*rctxp], smp_processor_id());
+	raw_data = this_cpu_ptr(perf_trace_buf[*rctxp]);
 
 	/* zero the dead bytes from align to not leak stack to user */
 	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
kernel/trace/trace_kprobe.c +2 -2
index faf7cef..f52b5f5 100644
@@ -1359,7 +1359,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
 }
 
@@ -1392,7 +1392,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
 	for (i = 0; i < tp->nr_args; i++)
 		call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
 
-	head = per_cpu_ptr(call->perf_events, smp_processor_id());
+	head = this_cpu_ptr(call->perf_events);
 	perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
 }
 
kernel/trace/trace_syscalls.c +2 -2
index d2c859c..34e3580 100644
@@ -519,7 +519,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
 			       (unsigned long *)&rec->args);
 
-	head = per_cpu_ptr(sys_data->enter_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->enter_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
@@ -595,7 +595,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	head = per_cpu_ptr(sys_data->exit_event->perf_events, smp_processor_id());
+	head = this_cpu_ptr(sys_data->exit_event->perf_events);
 	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
 }
 
#32 lore raw

[tip:perf/urgent] perf_events, trace: Fix perf_trace_destroy(), mutex went missing

In reply to Peter Zijlstra
Commit-ID:  2e97942fe57864588774f173cf4cd7bb68968b76
Gitweb:     http://git.kernel.org/tip/2e97942fe57864588774f173cf4cd7bb68968b76
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Fri, 21 May 2010 16:22:33 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 31 May 2010 08:46:09 +0200

perf_events, trace: Fix perf_trace_destroy(), mutex went missing

Steve spotted I forgot to do the destroy under event_mutex.

LKML-Reference: <1274451913.1674.1707.camel@laptop>

---
 kernel/trace/trace_event_perf.c |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)
kernel/trace/trace_event_perf.c +4 -1
index 49c7abf..e6f6588 100644
@@ -132,8 +132,9 @@ void perf_trace_destroy(struct perf_event *p_event)
 	struct ftrace_event_call *tp_event = p_event->tp_event;
 	int i;
 
+	mutex_lock(&event_mutex);
 	if (--tp_event->perf_refcount > 0)
-		return;
+		goto out;
 
 	if (tp_event->class->reg)
 		tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER);
@@ -157,6 +158,8 @@ void perf_trace_destroy(struct perf_event *p_event)
 			perf_trace_buf[i] = NULL;
 		}
 	}
+out:
+	mutex_unlock(&event_mutex);
 }
 
 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,