perf tracepoint and output optimizations
Subsystems (4)
PATCH 10/10 perf: Optimize perf_tp_event_match
In reply to Peter ZijlstraSince we know tracepoints come from kernel context, avoid conditionals that try and establish that very fact.
--- kernel/perf_event.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/perf_event.c +4 -1
@@ -4477,7 +4477,10 @@ static int perf_tp_event_match(struct pe struct perf_sample_data *data, struct pt_regs *regs) { - if (perf_exclude_event(event, regs)) +/* +* All tracepoints are from kernel-space. +*/ +if (event->attr.exclude_kernel)return 0 ;if (!perf_tp_filter_match (event, data))
PATCH 4/10 perf-record: Remove -M
In reply to Peter ZijlstraSince it is not allowed to create cross-cpu (or cross-task) buffers, this option is no longer valid.
--- tools/perf/builtin-record.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +12 -24
@@ -61,8 +61,6 @@ static bool call_graph = false;static bool inherit_stat = false;static bool no_samples = false;static bool sample_address = false; -static bool multiplex = false; -static int multiplex_fd =-1 ;static long samples =0 ;static struct timeval last_read;
@@ -381,27 +379,19 @@ try_again:*/ if (group && group_fd ==-1 ) group_fd = fd[nr_cpu][counter][thread_index]; -if (multiplex && multiplex_fd ==-1 ) - multiplex_fd = fd[nr_cpu][counter][thread_index]; -if (multiplex && fd[nr_cpu][counter][thread_index] != multiplex_fd) { - - ret =ioctl (fd[nr_cpu][counter][thread_index],PERF_EVENT_IOC_SET_OUTPUT , multiplex_fd); -assert (ret !=-1 ); - }else { - event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].events =POLLIN ; - nr_poll++; - - mmap_array[nr_cpu][counter][thread_index].counter = counter; - mmap_array[nr_cpu][counter][thread_index].prev =0 ; - mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; - mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, -PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); -if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { -error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); -exit (-1 ); - } + event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; + event_array[nr_poll].events =POLLIN ; + nr_poll++; + + mmap_array[nr_cpu][counter][thread_index].counter = counter; + mmap_array[nr_cpu][counter][thread_index].prev =0 ; + mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; + mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, +PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); +if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { +error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); +exit (-1 ); }if (filter !=NULL ) {
@@ -835,8 +825,6 @@ static const struct option options[] = { "Sample addresses"),OPT_BOOLEAN ('n' ,"no-samples" , &no_samples,"don't sample" ), -OPT_BOOLEAN ('M' ,"multiplex" , &multiplex, -"multiplex counter output in a single channel" ), OPT_END() };
PATCH 9/10 perf: Remove more fastpath code
In reply to Peter ZijlstraSanity checks cost instructions
--- include/linux/perf_event.h | 2 -- kernel/perf_event.c | 20 ++++---------------- 2 files changed, 4 insertions(+), 18 deletions(-)
include/linux/perf_event.h +0 -2
@@ -803,8 +803,6 @@ struct perf_cpu_context {struct perf_output_handle {struct perf_event *event;struct perf_mmap_data *data; -unsigned long head; -unsigned long offset;unsigned long wakeup;unsigned long size;void *addr;
kernel/perf_event.c +4 -16
@@ -2966,20 +2966,11 @@ again:preempt_enable (); } -void perf_output_copy (struct perf_output_handle *handle, +__always_inline void perf_output_copy(struct perf_output_handle *handle,const void *buf,unsigned int len) { - handle->offset += len; - -/* -* Check we didn't copy past our reservation window, taking the -* possible unsigned int wrap into account. -*/ -if (WARN_ON_ONCE (((long )(handle->head - handle->offset)) <0 )) -return ; - do { - unsigned long size = min(handle->size, len); + unsigned long size = min_t(unsigned long, handle->size, len); memcpy(handle->addr, buf, size);
@@ -3054,15 +3045,12 @@ int perf_output_begin(struct perf_output goto fail; }while (local_cmpxchg (&data->head, offset, head) != offset); - handle->offset = offset; - handle->head = head; -if (head -local_read (&data->wakeup) > data->watermark)local_add (data->watermark, &data->wakeup); - handle->page = handle->offset >> (PAGE_SHIFT +page_order (data)); + handle->page = offset >> (PAGE_SHIFT +page_order (data)); handle->page &= data->nr_pages -1 ; - handle->size = handle->offset & ((PAGE_SIZE <<page_order (data)) -1 ); + handle->size = offset & ((PAGE_SIZE <<page_order (data)) -1 ); handle->addr = data->data_pages[handle->page]; handle->addr += handle->size; handle->size = (PAGE_SIZE <<page_order (data)) - handle->size;
PATCH 7/10 perf: Optimize perf_output_copy
In reply to Peter ZijlstraReduce the clutter in perf_output_copy() by keeping an interator in perf_output_handle.
--- include/linux/perf_event.h | 3 ++ kernel/perf_event.c | 54 +++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 28 deletions(-)
include/linux/perf_event.h +3 -0
@@ -806,6 +806,9 @@ struct perf_output_handle {unsigned long head;unsigned long offset;unsigned long wakeup; +unsigned long size; +void *addr; +int page;int nmi;int sample; };
kernel/perf_event.c +26 -28
71 lines not on this page · raw message
PATCH 3/10 perf: Ensure IOC_OUTPUT isnt used to create multi-writer buffers
In reply to Peter ZijlstraSince we want to ensure buffers only have a single writer, we must avoid creating one with multiple.
--- kernel/perf_event.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+)
kernel/perf_event.c +19 -0
@@ -4885,6 +4885,13 @@ static int perf_event_set_output(structint fput_needed =0 ;int ret = -EINVAL ; +/* +* Don't allow output of inherited per-task events. This would +* create performance issues due to cross cpu access. +*/ +if (event->cpu ==-1 && event->attr.inherit) +return -EINVAL ; +if (!output_fd) goto set;
@@ -4905,6 +4912,18 @@ static int perf_event_set_output(structif (event->data) goto out; +/* +* Don't allow cross-cpu buffers +*/ +if (output_event->cpu != event->cpu) + goto out; + +/* +* If its not a per-cpu buffer, it must be the same task. +*/ +if (output_event->cpu ==-1 && output_event->ctx != event->ctx) + goto out; +atomic_long_inc (&output_file->f_count); set:
cover perf tracepoint and output optimizations
A number of patches that reworks the perf-tracepoint infrastructure and optimizes some of the generic perf output paths.
PATCH 8/10 perf: Optimize the !vmalloc backed buffer
Reduce code and data by using the knowledge that for !PERF_USE_VMALLOC data_order is always 0.
--- include/linux/perf_event.h | 2 +- kernel/perf_event.c | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 16 deletions(-)
include/linux/perf_event.h +1 -1
@@ -588,8 +588,8 @@ struct perf_mmap_data {struct rcu_head rcu_head;#ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; +int page_order;/* allocation order */ #endif -int data_order;/* allocation order */ int nr_pages;/* nr of data pages */ int writable;/* are we writable */ int nr_locked;/* nr pages mlocked */
kernel/perf_event.c +26 -15
102 lines not on this page · raw message
PATCH 1/10 perf, trace: Remove IRQ-disable from perf/tracepoint interaction
In reply to Peter ZijlstraLKML-Reference: <1274259525.5605.10352.camel@twins>
--- include/linux/ftrace_event.h | 9 +--- include/trace/ftrace.h | 17 +++------ kernel/trace/trace_event_perf.c | 73 +++++++++++++++------------------------- kernel/trace/trace_kprobe.c | 10 ++--- kernel/trace/trace_syscalls.c | 10 ++--- 5 files changed, 47 insertions(+), 72 deletions(-)
include/linux/ftrace_event.h +3 -6
@@ -197,20 +197,17 @@ extern void perf_trace_disable(int eventextern int ftrace_profile_set_filter (struct perf_event *event,int event_id,char *filter_str);extern void ftrace_profile_free_filter (struct perf_event *event); -extern void * -perf_trace_buf_prepare (int size,unsigned short type,int *rctxp, -unsigned long *irq_flags); +extern void *perf_trace_buf_prepare (int size,unsigned short type, +struct pt_regs *regs,int *rctxp);static inline void perf_trace_buf_submit (void *raw_data,int size,int rctx,u64 addr, -u64 count,unsigned long irq_flags,struct pt_regs *regs, -void *event) +u64 count,struct pt_regs *regs,void *event) {struct trace_entry *entry = raw_data;perf_tp_event (entry->type, addr, count, raw_data, size, regs, event);perf_swevent_put_recursion_context (rctx); -local_irq_restore (irq_flags); }#endif
include/trace/ftrace.h +7 -10
@@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_ev struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_raw_##call *entry; \ u64 __addr = 0, __count = 1; \ - unsigned long irq_flags; \ int __entry_size; \ int __data_size; \ int rctx; \
@@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_evif (WARN_ONCE (__entry_size >PERF_MAX_TRACE_SIZE , \"profile buffer not large enough" )) \return ; \ + \ entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare( \ - __entry_size, event_call->id, &rctx, &irq_flags); \ + __entry_size, event_call->id, __regs, &rctx); \ if (!entry) \ return; \ + \ tstruct \ \ { assign; } \ \ perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \ - __count, irq_flags, __regs, \ - event_call->perf_data); \ + __count, __regs, event_call->perf_data); \ }#undef DEFINE_EVENT
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev static notrace void perf_trace_##call(proto) \ { \ struct ftrace_event_call *event_call = &event_##call; \ - struct pt_regs *__regs = &get_cpu_var(perf_trace_regs); \ - \ - perf_fetch_caller_regs(__regs, 1); \ - \ - perf_trace_templ_##template(event_call, __regs, args); \ + struct pt_regs __regs; \ \ - put_cpu_var(perf_trace_regs); \ + perf_fetch_caller_regs(&__regs, 1); \ + perf_trace_templ_##template(event_call, &__regs, args); \ }#undef DEFINE_EVENT_PRINT
kernel/trace/trace_event_perf.c +29 -44
154 lines not on this page · raw message
kernel/trace/trace_kprobe.c +4 -6
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(sstruct kprobe_trace_entry_head *entry;u8 *data;int size, __size, i; -unsigned long irq_flags;int rctx; __size =sizeof (*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(s "profile buffer not large enough"))return ; - entry =perf_trace_buf_prepare (size, call->id, &rctx, &irq_flags); + entry =perf_trace_buf_prepare (size, call->id, regs, &rctx);if (!entry)return ;
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(sfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , irq_flags, regs, call->perf_data); +perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, call->perf_data); }/* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_funstruct kretprobe_trace_entry_head *entry;u8 *data;int size, __size, i; -unsigned long irq_flags;int rctx; __size =sizeof (*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_fun "profile buffer not large enough"))return ; - entry =perf_trace_buf_prepare (size, call->id, &rctx, &irq_flags); + entry =perf_trace_buf_prepare (size, call->id, regs, &rctx);if (!entry)return ;
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_funcall_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset);perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , - irq_flags, regs, call->perf_data); + regs, call->perf_data); }static int probe_perf_enable (struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +4 -6
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt {struct syscall_metadata *sys_data;struct syscall_trace_enter *rec; -unsigned long flags;int syscall_nr;int rctx;int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct ptreturn ; rec = (struct syscall_trace_enter *)perf_trace_buf_prepare (size, - sys_data->enter_event->id, &rctx, &flags); + sys_data->enter_event->id, regs, &rctx);if (!rec)return ; rec->nr = syscall_nr;syscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , flags, regs, +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, sys_data->enter_event->perf_data); }
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_ {struct syscall_metadata *sys_data;struct syscall_trace_exit *rec; -unsigned long flags;int syscall_nr;int rctx;int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_return ; rec = (struct syscall_trace_exit *)perf_trace_buf_prepare (size, - sys_data->exit_event->id, &rctx, &flags); + sys_data->exit_event->id, regs, &rctx);if (!rec)return ; rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , flags, regs, +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, sys_data->exit_event->perf_data); }
PATCH 6/10 perf: Fix wakeup storm for RO mmap()s
In reply to Peter ZijlstraRO mmap()s don't update the tail pointer, so comparing against it for determining the written data size doesn't really do any good.
Keep track of when we last did a wakeup, and compare against that.
--- include/linux/perf_event.h | 2 +- kernel/perf_event.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
include/linux/perf_event.h +1 -1
@@ -599,7 +599,7 @@ struct perf_mmap_data {local_t head;/* write position */ local_t nest;/* nested writers */ local_t events;/* event limit */ -local_t wakeup;/* needs a wakeup */ +local_t wakeup;/* wakeup stamp */ local_t lost;/* nr records lost */ long watermark;/* wakeup watermark */
kernel/perf_event.c +2 -2
@@ -3055,8 +3055,8 @@ int perf_output_begin(struct perf_output handle->offset = offset; handle->head = head; -if (head - tail > data->watermark) -local_inc (&data->wakeup); +if (head -local_read (&data->wakeup) > data->watermark) +local_add (data->watermark, &data->wakeup);if (have_lost) { lost_event.header.type =PERF_RECORD_LOST ;
PATCH 2/10 perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraAvoid the swevent hash-table by using per-tracepoint hlists.
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
--- include/linux/ftrace_event.h | 16 ++--- include/linux/perf_event.h | 6 + include/trace/ftrace.h | 4 - kernel/perf_event.c | 94 +++++++++++++++-------------- kernel/trace/trace_event_perf.c | 127 +++++++++++++++++++++------------------- kernel/trace/trace_kprobe.c | 9 +- kernel/trace/trace_syscalls.c | 11 ++- 7 files changed, 143 insertions(+), 124 deletions(-)
include/linux/ftrace_event.h +8 -8
@@ -133,7 +133,7 @@ struct ftrace_event_call {void *data;int perf_refcount; -void *perf_data; +struct hlist_head *perf_events;int (*perf_event_enable)(struct ftrace_event_call *);void (*perf_event_disable)(struct ftrace_event_call *); };
@@ -192,9 +192,11 @@ struct perf_event; DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); -extern int perf_trace_enable (int event_id,void *data); -extern void perf_trace_disable (int event_id); -extern int ftrace_profile_set_filter (struct perf_event *event,int event_id, +extern int perf_trace_init (struct perf_event *event); +extern void perf_trace_destroy (struct perf_event *event); +extern int perf_trace_enable (struct perf_event *event); +extern void perf_trace_disable (struct perf_event *event); +extern int ftrace_profile_set_filter (struct perf_event *event,int event_id,char *filter_str);extern void ftrace_profile_free_filter (struct perf_event *event); extern void *perf_trace_buf_prepare(int size, unsigned short type,
@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(intstatic inline void perf_trace_buf_submit (void *raw_data,int size,int rctx,u64 addr, -u64 count,struct pt_regs *regs,void *event) +u64 count,struct pt_regs *regs,void *head) { -struct trace_entry *entry = raw_data; - -perf_tp_event (entry->type, addr, count, raw_data, size, regs, event); +perf_tp_event (addr, count, raw_data, size, regs, head);perf_swevent_put_recursion_context (rctx); }#endif
include/trace/ftrace.h +3 -1
@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_ev struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_raw_##call *entry; \ u64 __addr = 0, __count = 1; \ + struct hlist_head *head; \ int __entry_size; \ int __data_size; \ int rctx; \
@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_ev \ { assign; } \ \ + head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ - __count, __regs, event_call->perf_data); \ + __count, __regs, head); \ }#undef DEFINE_EVENT
kernel/trace/trace_event_perf.c +67 -60
197 lines not on this page · raw message
kernel/trace/trace_kprobe.c +6 -3
@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(s struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);struct ftrace_event_call *call = &tp->call;struct kprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(sfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ()); +perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }/* Kretprobe profile handler */
@@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_fun struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);struct ftrace_event_call *call = &tp->call;struct kretprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_funfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , - regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ()); +perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }static int probe_perf_enable (struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +7 -4
@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt {struct syscall_metadata *sys_data;struct syscall_trace_enter *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt rec->nr = syscall_nr;syscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->enter_event->perf_data); + + head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ()); +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysenter_enable (struct ftrace_event_call *call)
@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_ {struct syscall_metadata *sys_data;struct syscall_trace_exit *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_ rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->exit_event->perf_data); + head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ()); +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysexit_enable (struct ftrace_event_call *call)
kernel/perf_event.c +48 -46
173 lines not on this page · raw message
include/linux/perf_event.h +4 -2
@@ -727,6 +727,7 @@ struct perf_event {perf_overflow_handler_t overflow_handler;#ifdef CONFIG_EVENT_TRACING +struct ftrace_event_call *tp_event;struct event_filter *filter;#endif
@@ -992,8 +993,9 @@ static inline bool perf_paranoid_kernel( }extern void perf_event_init (void ); -extern void perf_tp_event (int event_id,u64 addr,u64 count,void *record, -int entry_size,struct pt_regs *regs,void *event); +extern void perf_tp_event (u64 addr,u64 count,void *record, +int entry_size,struct pt_regs *regs, +struct hlist_head *head);extern void perf_bp_event (struct perf_event *event,void *data);#ifndef perf_misc_flags
PATCH 5/10 perf-record: Share per-cpu buffers
In reply to Peter ZijlstraIt seems a waste of space to create a buffer per event, share it per-cpu.
--- tools/perf/builtin-record.c | 52 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +28 -24
@@ -85,7 +85,7 @@ struct mmap_data {unsigned int prev; }; -static struct mmap_data *mmap_array[MAX_NR_CPUS ][MAX_COUNTERS ]; +static struct mmap_data mmap_array[MAX_NR_CPUS ];static unsigned long mmap_read_head (struct mmap_data *md) {
@@ -380,18 +380,29 @@ try_again:if (group && group_fd ==-1 ) group_fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].events =POLLIN ; - nr_poll++; - - mmap_array[nr_cpu][counter][thread_index].counter = counter; - mmap_array[nr_cpu][counter][thread_index].prev =0 ; - mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; - mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, -PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); -if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { -error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); -exit (-1 ); +if (counter || thread_index) { + ret =ioctl (fd[nr_cpu][counter][thread_index], +PERF_EVENT_IOC_SET_OUTPUT , + fd[nr_cpu][0 ][0 ]); +if (ret) { +error ("failed to set output: %d (%s)\n" , errno, +strerror (errno)); +exit (-1 ); + } + }else { + mmap_array[nr_cpu].counter = counter; + mmap_array[nr_cpu].prev =0 ; + mmap_array[nr_cpu].mask = mmap_pages*page_size -1 ; + mmap_array[nr_cpu].base =mmap (NULL , (mmap_pages+1 )*page_size, +PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); +if (mmap_array[nr_cpu].base ==MAP_FAILED ) { +error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); +exit (-1 ); + } + + event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; + event_array[nr_poll].events =POLLIN ; + nr_poll++; }if (filter !=NULL ) {
@@ -492,16 +503,11 @@ static struct perf_event_header finishedstatic void mmap_read_all (void ) { -int i, counter, thread; +int i;for (i =0 ; i < nr_cpu; i++) { -for (counter =0 ; counter < nr_counters; counter++) { -for (thread =0 ; thread < thread_num; thread++) { -if (mmap_array[i][counter][thread].base) -mmap_read (&mmap_array[i][counter][thread]); - } - - } +if (mmap_array[i].base) +mmap_read (&mmap_array[i]); }if (perf_header__has_feat (&session->header,HEADER_TRACE_INFO ))
@@ -876,9 +882,7 @@ int cmd_record(int argc, const char **arfor (i =0 ; i <MAX_NR_CPUS ; i++) {for (j =0 ; j <MAX_COUNTERS ; j++) { fd[i][j] =malloc (sizeof (int )*thread_num); - mmap_array[i][j] =zalloc ( -sizeof (struct mmap_data )*thread_num); -if (!fd[i][j] || !mmap_array[i][j]) +if (!fd[i][j])return -ENOMEM ; } }
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraQuoted 71 lines · Peter Zijlstra
Avoid the swevent hash-table by using per-tracepoint hlists.
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
Signed-off-by Peter Zijlstra <[email protected]>--- include/linux/ftrace_event.h | 16 ++--- include/linux/perf_event.h | 6 + include/trace/ftrace.h | 4 - kernel/perf_event.c | 94 +++++++++++++++-------------- kernel/trace/trace_event_perf.c | 127 +++++++++++++++++++++------------------- kernel/trace/trace_kprobe.c | 9 +- kernel/trace/trace_syscalls.c | 11 ++- 7 files changed, 143 insertions(+), 124 deletions(-)Index:
linux-2.6/include/linux/ftrace_event.h
===================================================================include/linux/ftrace_event.h +8
-8@@ -133,7 +133,7 @@ struct ftrace_event_call {void *data;int perf_refcount; -void *perf_data; +struct hlist_head *perf_events;int (*perf_event_enable)(struct ftrace_event_call *);void (*perf_event_disable)(struct ftrace_event_call *); };@@ -192,9 +192,11 @@ struct perf_event; DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); -extern int perf_trace_enable (int event_id,void *data); -extern void perf_trace_disable (int event_id); -extern int ftrace_profile_set_filter (struct perf_event *event,int event_id, +extern int perf_trace_init (struct perf_event *event); +extern void perf_trace_destroy (struct perf_event *event); +extern int perf_trace_enable (struct perf_event *event); +extern void perf_trace_disable (struct perf_event *event); +extern int ftrace_profile_set_filter (struct perf_event *event,int event_id,char *filter_str);extern void ftrace_profile_free_filter (struct perf_event *event); extern void *perf_trace_buf_prepare(int size, unsigned short type,@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(intstatic inline void perf_trace_buf_submit (void *raw_data,int size,int rctx,u64 addr, -u64 count,struct pt_regs *regs,void *event) +u64 count,struct pt_regs *regs,void *head) { -struct trace_entry *entry = raw_data; - -perf_tp_event (entry->type, addr, count, raw_data, size, regs, event); +perf_tp_event (addr, count, raw_data, size, regs, head);perf_swevent_put_recursion_context (rctx); }#endif Index:
linux-2.6/include/trace/ftrace.h
===================================================================include/trace/ftrace.h +1
-0@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_ev struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_raw_##call *entry; \ u64 __addr = 0, __count = 1; \ + struct hlist_head *head; \ int __entry_size; \ int __data_size; \ int rctx; \
include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794
Quoted 4 lines
include/trace/ftrace.h +1
-0@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_ev \ { assign; } \ \ + head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\
Should be rcu_dereference_sched ?
Quoted 27 lines
include/trace/ftrace.h +1
-1perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ - __count, __regs, event_call->perf_data); \ + __count, __regs, head); \ }#undef DEFINE_EVENTIndex:
linux-2.6/kernel/trace/trace_event_perf.c
===================================================================kernel/trace/trace_event_perf.c +6
-3@@ -23,14 +23,25 @@ typedef typeof(unsigned long [PERF_MAX_T/* Count the events in use (per event id, not per instance) */ static int total_ref_count; -static int perf_trace_event_enable (struct ftrace_event_call *event,void *data) +static int perf_trace_event_init (struct ftrace_event_call *tp_event, +struct perf_event *p_event) { +struct hlist_head *list;int ret = -ENOMEM ; +int cpu; -if (event->perf_refcount++ >0 ) { - event->perf_data =NULL ; + p_event->tp_event = tp_event; +if (tp_event->perf_refcount++ >0 )return 0 ;
kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44
Quoted 10 lines
kernel/trace/trace_event_perf.c +9
-1- } + + list = alloc_percpu(struct hlist_head); +if (!list) + goto fail; + +for_each_possible_cpu (cpu) +INIT_HLIST_HEAD (per_cpu_ptr (list, cpu)); + + tp_event->perf_events = list;
I suspect this must be rcu_assign_pointer.
Quoted 84 lines
kernel/trace/trace_event_perf.c +27
-20if (!total_ref_count) {char *buf;@@ -39,20 +50,20 @@ static int perf_trace_event_enable(strucfor (i =0 ; i <4 ; i++) { buf = (char *)alloc_percpu (perf_trace_t);if (!buf) - goto fail_buf; + goto fail; -rcu_assign_pointer (perf_trace_buf[i], buf); + perf_trace_buf[i] = buf; } } - ret = event->perf_event_enable (event); -if (!ret) { - event->perf_data = data; - total_ref_count++; -return 0 ; - } + ret = tp_event->perf_event_enable (tp_event); +if (ret) + goto fail; -fail_buf: + total_ref_count++; +return 0 ; + +fail:if (!total_ref_count) {int i;@@ -61,21 +72,26 @@ fail_buf: perf_trace_buf[i] =NULL ; } } - event->perf_refcount--; + +if (!--tp_event->perf_refcount) { +free_percpu (tp_event->perf_events); + tp_event->perf_events =NULL ; + }return ret; } -int perf_trace_enable (int event_id,void *data) +int perf_trace_init (struct perf_event *p_event) { -struct ftrace_event_call *event; +struct ftrace_event_call *tp_event; +int event_id = p_event->attr.config;int ret = -EINVAL ;mutex_lock (&event_mutex); -list_for_each_entry (event, &ftrace_events, list) { -if (event->id == event_id && event->perf_event_enable && -try_module_get (event->mod)) { - ret =perf_trace_event_enable (event, data); +list_for_each_entry (tp_event, &ftrace_events, list) { +if (tp_event->id == event_id && tp_event->perf_event_enable && +try_module_get (tp_event->mod)) { + ret =perf_trace_event_init (tp_event, p_event);break ; } }@@ -84,53 +100,52 @@ int perf_trace_enable(int event_id, voidreturn ret; } -static void perf_trace_event_disable (struct ftrace_event_call *event) +int perf_trace_enable (struct perf_event *p_event) { -if (--event->perf_refcount >0 ) -return ; +struct ftrace_event_call *tp_event = p_event->tp_event; +struct hlist_head *list; - event->perf_event_disable (event); + list = tp_event->perf_events; +if (WARN_ON_ONCE (!list)) +return -EINVAL ;
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113
Quoted 10 lines
kernel/trace/trace_event_perf.c +2
-8-if (!--total_ref_count) { -char *buf[4 ]; -int i; - -for (i =0 ; i <4 ; i++) { - buf[i] = perf_trace_buf[i]; -rcu_assign_pointer (perf_trace_buf[i],NULL ); - } + list =per_cpu_ptr (list,smp_processor_id ()); +hlist_add_head_rcu (&p_event->hlist_entry, list);
Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)
Quoted 23 lines
kernel/trace/trace_event_perf.c +8
-10-/* -* Ensure every events in profiling have finished before -* releasing the buffers -*/ -synchronize_sched (); +return 0 ; +} -for (i =0 ; i <4 ; i++) -free_percpu (buf[i]); - } +void perf_trace_disable (struct perf_event *p_event) +{ +hlist_del_rcu (&p_event->hlist_entry); } -void perf_trace_disable (int event_id) +void perf_trace_destroy (struct perf_event *p_event) { -struct ftrace_event_call *event; +struct ftrace_event_call *tp_event = p_event->tp_event; +int i;
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131
Quoted 10 lines
kernel/trace/trace_event_perf.c +4
-6-mutex_lock (&event_mutex); -list_for_each_entry (event, &ftrace_events, list) { -if (event->id == event_id) { -perf_trace_event_disable (event); -module_put (event->mod); -break ; +if (--tp_event->perf_refcount >0 ) +return ; + + tp_event->perf_event_disable (tp_event);
Don't we need a rcu_synchronize_sched() here?
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 132
Quoted 2 lines
kernel/trace/trace_event_perf.c +2
-0+free_percpu (tp_event->perf_events); + tp_event->perf_events =NULL ;
And rcu_assign?
Quoted 22 lines
kernel/trace/trace_event_perf.c +6
-2+ +if (!--total_ref_count) { +for (i =0 ; i <4 ; i++) { +free_percpu (perf_trace_buf[i]); + perf_trace_buf[i] =NULL ; } } -mutex_unlock (&event_mutex); }__kprobes void *perf_trace_buf_prepare (int size,unsigned short type,struct pt_regs *regs,int *rctxp) {struct trace_entry *entry; -char *trace_buf, *raw_data; +char *raw_data;int pc;BUILD_BUG_ON (PERF_MAX_TRACE_SIZE %sizeof (unsigned long ));@@ -139,13 +154,9 @@ __kprobes void *perf_trace_buf_prepare(i *rctxp =perf_swevent_get_recursion_context ();
kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159
Quoted 10 lines
kernel/trace/trace_event_perf.c +2
-6if (*rctxp <0 ) - goto err_recursion; - - trace_buf =rcu_dereference_sched (perf_trace_buf[*rctxp]); -if (!trace_buf) - goto err; +return NULL ; - raw_data =per_cpu_ptr (trace_buf,smp_processor_id ()); + raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ());
Needs rcu_dereference_sched too. And this could be __this_cpu_var()
Quoted 26 lines
kernel/trace/trace_event_perf.c +0
-4/* zero the dead bytes from align to not leak stack to user */ memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64));@@ -155,9 +166,5 @@ __kprobes void *perf_trace_buf_prepare(i entry->type = type;return raw_data; -err: -perf_swevent_put_recursion_context (*rctxp); -err_recursion: -return NULL ; }EXPORT_SYMBOL_GPL (perf_trace_buf_prepare);Index:
linux-2.6/kernel/trace/trace_kprobe.c
===================================================================kernel/trace/trace_kprobe.c +1
-0@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(s struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);struct ftrace_event_call *call = &tp->call;struct kprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
kernel/trace/trace_kprobe.c static __kprobes void kprobe_perf_func(s · line 1365
Quoted 5 lines
kernel/trace/trace_kprobe.c +1
-1@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(sfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ());
Same
Quoted 13 lines
kernel/trace/trace_kprobe.c +2
-0+perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }/* Kretprobe profile handler */ @@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_fun struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);struct ftrace_event_call *call = &tp->call;struct kretprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
kernel/trace/trace_kprobe.c static __kprobes void kretprobe_perf_fun · line 1398
Quoted 6 lines
kernel/trace/trace_kprobe.c +1
-2@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_funfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , - regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ());
ditto
Quoted 17 lines
kernel/trace/trace_kprobe.c +1
-0+perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }static int probe_perf_enable (struct ftrace_event_call *call)Index:
linux-2.6/kernel/trace/trace_syscalls.c
===================================================================kernel/trace/trace_syscalls.c +1
-0@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt {struct syscall_metadata *sys_data;struct syscall_trace_enter *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
kernel/trace/trace_syscalls.c static void perf_syscall_enter(struct pt · line 472
Quoted 7 lines
kernel/trace/trace_syscalls.c +2
-2@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt rec->nr = syscall_nr;syscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->enter_event->perf_data); + + head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ());
ditto
Quoted 13 lines
kernel/trace/trace_syscalls.c +2
-0+perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysenter_enable (struct ftrace_event_call *call)@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_ {struct syscall_metadata *sys_data;struct syscall_trace_exit *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
kernel/trace/trace_syscalls.c static void perf_syscall_exit(struct pt_ · line 548
Quoted 6 lines
kernel/trace/trace_syscalls.c +1
-2@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_ rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->exit_event->perf_data); + head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ());
and ditto :)
Quoted 59 lines
kernel/trace/trace_syscalls.c +1
-0+perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysexit_enable (struct ftrace_event_call *call)Index:
linux-2.6/kernel/perf_event.c
===================================================================kernel/perf_event.c +2
-12@@ -4004,9 +4004,6 @@ static void perf_swevent_add(struct perfperf_swevent_overflow (event,0 , nmi, data, regs); } -static int perf_tp_event_match (struct perf_event *event, -struct perf_sample_data *data); -static int perf_exclude_event (struct perf_event *event,struct pt_regs *regs) {@@ -4036,10 +4033,6 @@ static int perf_swevent_match(struct perif (perf_exclude_event (event, regs))return 0 ; -if (event->attr.type ==PERF_TYPE_TRACEPOINT && - !perf_tp_event_match (event, data)) -return 0 ; -return 1 ; }@@ -4094,7 +4087,7 @@ end:int perf_swevent_get_recursion_context (void ) { -struct perf_cpu_context *cpuctx = &get_cpu_var (perf_cpu_context); +struct perf_cpu_context *cpuctx = &__get_cpu_var (perf_cpu_context);int rctx;if (in_nmi ())@@ -4106,10 +4099,8 @@ int perf_swevent_get_recursion_context(velse rctx =0 ; -if (cpuctx->recursion[rctx]) { -put_cpu_var (perf_cpu_context); +if (cpuctx->recursion[rctx])return -1 ; - } cpuctx->recursion[rctx]++;barrier ();@@ -4123,7 +4114,6 @@ void perf_swevent_put_recursion_context(struct perf_cpu_context *cpuctx = &__get_cpu_var (perf_cpu_context);barrier (); cpuctx->recursion[rctx]--; -put_cpu_var (perf_cpu_context); }EXPORT_SYMBOL_GPL (perf_swevent_put_recursion_context);
kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127
Quoted 4 lines
kernel/perf_event.c +1
-0@@ -4134,6 +4124,7 @@ void __perf_sw_event(u32 event_id, u64 nstruct perf_sample_data data;int rctx; +preempt_disable_notrace ();
Why is this needed. We have the recursion context protection already.
Ok, otherwise looks good.
Re: [PATCH 05/10] perf-record: Share per-cpu buffers
In reply to Peter ZijlstraQuoted 39 lines · Peter Zijlstra
It seems a waste of space to create a buffer per event, share it per-cpu.
Signed-off-by Peter Zijlstra <[email protected]>--- tools/perf/builtin-record.c | 52 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 24 deletions(-)Index:
linux-2.6/tools/perf/builtin-record.c
===================================================================tools/perf/builtin-record.c +4
-13@@ -85,7 +85,7 @@ struct mmap_data {unsigned int prev; }; -static struct mmap_data *mmap_array[MAX_NR_CPUS ][MAX_COUNTERS ]; +static struct mmap_data mmap_array[MAX_NR_CPUS ];static unsigned long mmap_read_head (struct mmap_data *md) {@@ -380,18 +380,29 @@ try_again:if (group && group_fd ==-1 ) group_fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].events =POLLIN ; - nr_poll++; - - mmap_array[nr_cpu][counter][thread_index].counter = counter; - mmap_array[nr_cpu][counter][thread_index].prev =0 ; - mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; - mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, -PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); -if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { -error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); -exit (-1 ); +if (counter || thread_index) { + ret = ioctl(fd[nr_cpu][counter][thread_index], + PERF_EVENT_IOC_SET_OUTPUT,
tools/perf/builtin-record.c try_again: · line 395
Quoted 10 lines
tools/perf/builtin-record.c +10
-0+ fd[nr_cpu][0][0]); +if (ret) { +error ("failed to set output: %d (%s)\n" , errno, +strerror (errno)); +exit (-1 ); + } + }else { + mmap_array[nr_cpu].counter = counter; + mmap_array[nr_cpu].prev =0 ; + mmap_array[nr_cpu].mask = mmap_pages*page_size -1 ;
If you do this multiplex per cpu (which I think is a very good thing), you should also increase the default value of mmap_pages as it might not be big enough anymore.
Quoted 46 lines
tools/perf/builtin-record.c +14
-11+ mmap_array[nr_cpu].base =mmap (NULL , (mmap_pages+1 )*page_size, +PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); +if (mmap_array[nr_cpu].base ==MAP_FAILED ) { +error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); +exit (-1 ); + } + + event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; + event_array[nr_poll].events =POLLIN ; + nr_poll++; }if (filter !=NULL ) {@@ -492,16 +503,11 @@ static struct perf_event_header finishedstatic void mmap_read_all (void ) { -int i, counter, thread; +int i;for (i =0 ; i < nr_cpu; i++) { -for (counter =0 ; counter < nr_counters; counter++) { -for (thread =0 ; thread < thread_num; thread++) { -if (mmap_array[i][counter][thread].base) -mmap_read (&mmap_array[i][counter][thread]); - } - - } +if (mmap_array[i].base) +mmap_read (&mmap_array[i]); }if (perf_header__has_feat (&session->header,HEADER_TRACE_INFO ))@@ -876,9 +882,7 @@ int cmd_record(int argc, const char **arfor (i =0 ; i <MAX_NR_CPUS ; i++) {for (j =0 ; j <MAX_COUNTERS ; j++) { fd[i][j] =malloc (sizeof (int )*thread_num); - mmap_array[i][j] =zalloc ( -sizeof (struct mmap_data )*thread_num); -if (!fd[i][j] || !mmap_array[i][j]) +if (!fd[i][j])return -ENOMEM ; } }
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Frederic WeisbeckerQuoted 9 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
\include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794
Quoted 1 line
include/trace/ftrace.h +1
-0+ head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\Should be
rcu_dereference_sched?
No, I removed all that rcu stuff and synchronized against the probe unregister.
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
Quoted 8 lines
kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44
Quoted 4 lines
kernel/trace/trace_event_perf.c +4
-0+for_each_possible_cpu (cpu) +INIT_HLIST_HEAD (per_cpu_ptr (list, cpu)); + + tp_event->perf_events = list;I suspect this must be
rcu_assign_pointer.
Same thing as above, I do this before probe register, so I see no need for RCU.
Quoted 7 lines
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113
Quoted 2 lines
kernel/trace/trace_event_perf.c +2
-0+ list =per_cpu_ptr (list,smp_processor_id ()); +hlist_add_head_rcu (&p_event->hlist_entry, list);Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)
What exactly is the puzzlement about?
Quoted 8 lines
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131
Quoted 4 lines
kernel/trace/trace_event_perf.c +4
-0+if (--tp_event->perf_refcount >0 ) +return ; + + tp_event->perf_event_disable (tp_event);Don't we need a
rcu_synchronize_sched()here?
Doesn't probe unregister synchronize things against its own callback?
Quoted 6 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ free_percpu(tp_event->perf_events); + tp_event->perf_events = NULL;And
rcu_assign?
Which again, makes any use of RCU unneeded.
Quoted 5 lines
kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159
Quoted 1 line
kernel/trace/trace_event_perf.c +1
-0+ raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ());Needs
rcu_dereference_schedtoo. And this could be__this_cpu_var()
Ahh! so that is what its called.
Quoted 4 lines
kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127
Quoted 1 line
kernel/perf_event.c +1
-0+preempt_disable_notrace ();Why is this needed. We have the recursion context protection already.
Because:
@@ -4094,7 +4087,7 @@ end:
int perf_swevent_get_recursion_context(void)
{
- struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+ struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
int rctx;
if (in_nmi())Peter Zijlstra
maintainerRe: [PATCH 05/10] perf-record: Share per-cpu buffers
In reply to Frederic WeisbeckerQuoted 3 lines · Frederic Weisbecker
If you do this multiplex per cpu (which I think is a very good thing), you should also increase the default value of
mmap_pagesas it might not be big enough anymore.
Hmm, I didn't notice any problems with that.
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraQuoted 11 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 3 lines · Peter Zijlstra
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
\include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 794
Quoted 1 line
include/trace/ftrace.h +1
-0+ head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\Should be
rcu_dereference_sched?
Quoted 5 lines
No, I removed all that rcu stuff and synchronized against the probe unregister.
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
I'm not sure about this. The tracepoints are called under rcu_read_lock(), but there is not synchronize_rcu() after we unregister a tracepoint, which means you can have a pending preempted one somewhere.
There is a call_rcu that removes the callbacks, but that only protect the callback themselves.
Quoted 21 lines
Quoted 9 lines
kernel/trace/trace_event_perf.c typedef typeof(unsigned long [PERF_MAX_T · line 44
Quoted 5 lines
kernel/trace/trace_event_perf.c +4
-0+for_each_possible_cpu (cpu) +INIT_HLIST_HEAD (per_cpu_ptr (list, cpu)); + + tp_event->perf_events = list;I suspect this must be
rcu_assign_pointer.Same thing as above, I do this before probe register, so I see no need for RCU.
Quoted 7 lines
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 113
Quoted 2 lines
kernel/trace/trace_event_perf.c +2
-0+ list =per_cpu_ptr (list,smp_processor_id ()); +hlist_add_head_rcu (&p_event->hlist_entry, list);Ah and may be small comment, because using the hlist api here may puzzle more people than just me ;)
Quoted 1 line
What exactly is the puzzlement about?
The fact we use the hlist API not for hlist purpose but for a list.
Quoted 10 lines
Quoted 8 lines
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 131
Quoted 4 lines
kernel/trace/trace_event_perf.c +4
-0+if (--tp_event->perf_refcount >0 ) +return ; + + tp_event->perf_event_disable (tp_event);Don't we need a
rcu_synchronize_sched()here?Doesn't probe unregister synchronize things against its own callback?
May be I missed it but it doesn't seem so.
Quoted 7 lines
Quoted 5 lines
kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(i · line 159
Quoted 1 line
kernel/trace/trace_event_perf.c +1
-0+ raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ());Needs
rcu_dereference_schedtoo. And this could be__this_cpu_var()Ahh! so that is what its called.
:)
Quoted 9 lines
Quoted 4 lines
kernel/perf_event.c void __perf_sw_event(u32 event_id, u64 n · line 4127
Quoted 1 line
kernel/perf_event.c +1
-0+preempt_disable_notrace ();Why is this needed. We have the recursion context protection already.
Because:
kernel/perf_event.c end: · line 4093
Quoted 8 lines
kernel/perf_event.c +1
-1@@ -4094,7 +4087,7 @@ end:int perf_swevent_get_recursion_context (void ) { -struct perf_cpu_context *cpuctx = &get_cpu_var (perf_cpu_context); +struct perf_cpu_context *cpuctx = &__get_cpu_var (perf_cpu_context);int rctx;if (in_nmi ())
Right.
Thanks.
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 3 lines · Frederic Weisbecker
Quoted 1 line
What exactly is the puzzlement about?
The fact we use the hlist API not for hlist purpose but for a list.
I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct list_head stuff.
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
Quoted 1 line
What exactly is the puzzlement about?
The fact we use the hlist API not for hlist purpose but for a list.
I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct
list_headstuff.
It's just that people might except this is used for a hlist purpose.
But nevermind, that's just a small issue.
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Frederic WeisbeckerQuoted 9 lines · Frederic Weisbecker
Quoted 2 lines
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
I'm not sure about this. The tracepoints are called under
rcu_read_lock(), but there is notsynchronize_rcu()after we unregister a tracepoint, which means you can have a pending preempted one somewhere.There is a
call_rcuthat removes the callbacks, but that only protect the callback themselves.
Ah, ok, so we should do probe_unregister + synchronize_sched().
That should ensure __DO_TRACE() doesn't call into it anymore.
/me goes make a patch
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 8 lines · Peter Zijlstra
Quoted 7 lines · Frederic Weisbecker
Quoted 2 lines
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
I'm not sure about this. The tracepoints are called under
rcu_read_lock(), but there is notsynchronize_rcu()after we unregister a tracepoint, which means you can have a pending preempted one somewhere.
Quoted 8 lines
Quoted 2 lines
There is a
call_rcuthat removes the callbacks, but that only protect the callback themselves.Ah, ok, so we should do
probe_unregister+synchronize_sched(). That should ensure__DO_TRACE()doesn't call into it anymore./me goes make a patch
Yep. But that also means we need to rcu_dereference_sched() to access the per cpu list of events.
Thanks.
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 15 lines · Frederic Weisbecker
Quoted 14 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 2 lines
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
I'm not sure about this. The tracepoints are called under
rcu_read_lock(), but there is notsynchronize_rcu()after we unregister a tracepoint, which means you can have a pending preempted one somewhere.There is a
call_rcuthat removes the callbacks, but that only protect the callback themselves.Ah, ok, so we should do
probe_unregister+synchronize_sched(). That should ensure__DO_TRACE()doesn't call into it anymore.
Quoted 6 lines
Quoted 2 lines
/me goes make a patch
Yep. But that also means we need to
rcu_dereference_sched()to access the per cpu list of events.
Why?
The per-cpu vars are allocated and freed in a fully serialized manner, there should be no races what so ever.
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 20 lines · Peter Zijlstra
Quoted 19 lines · Frederic Weisbecker
Quoted 16 lines · Peter Zijlstra
Quoted 9 lines · Frederic Weisbecker
Quoted 2 lines
I assumed that after probe unregister a tracepoint callback doesn't happen, which then guarantees we should never get !head.
I'm not sure about this. The tracepoints are called under
rcu_read_lock(), but there is notsynchronize_rcu()after we unregister a tracepoint, which means you can have a pending preempted one somewhere.There is a
call_rcuthat removes the callbacks, but that only protect the callback themselves.Ah, ok, so we should do
probe_unregister+synchronize_sched(). That should ensure__DO_TRACE()doesn't call into it anymore./me goes make a patch
Quoted 7 lines
Quoted 2 lines
Yep. But that also means we need to
rcu_dereference_sched()to access the per cpu list of events.Why?
The per-cpu vars are allocated and freed in a fully serialized manner, there should be no races what so ever.
Ah right, I was confused.
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line
What exactly is the puzzlement about?
The fact we use the hlist API not for hlist purpose but for a list.
I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct
list_headstuff.
Using an API in such a mixed way may cause puzzlement ;-)
Fortunately we've got the ultimate anti-puzzlement weapon:
code comments.
Ingo
Peter Zijlstra
maintainerPATCH 1/1 perf, trace: Fix probe unregister race
In reply to Peter Zijlstratracepoint_probe_unregister() does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.
Also, use this_cpu_ptr() where possible.
--- include/trace/ftrace.h | 2 +- kernel/trace/trace_event_perf.c | 10 ++++++++-- kernel/trace/trace_kprobe.c | 4 ++-- kernel/trace/trace_syscalls.c | 4 ++-- 4 files changed, 13 insertions(+), 7 deletions(-)
include/trace/ftrace.h +1 -1
@@ -791,7 +791,7 @@ perf_trace_templ_##call(struct ftrace_ev \ { assign; } \ \ - head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\ + head =this_cpu_ptr (event_call->perf_events); \perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ __count, __regs, head); \ }
kernel/trace/trace_event_perf.c +8 -2
@@ -109,7 +109,7 @@ int perf_trace_enable(struct perf_eventif (WARN_ON_ONCE (!list))return -EINVAL ; - list =per_cpu_ptr (list,smp_processor_id ()); + list =this_cpu_ptr (list);hlist_add_head_rcu (&p_event->hlist_entry, list);return 0 ;
@@ -130,6 +130,12 @@ void perf_trace_destroy(struct perf_even tp_event->perf_event_disable (tp_event); +/* +* Ensure our callback won't be called anymore. See +* tracepoint_probe_unregister() and __DO_TRACE(). +*/ +synchronize_sched (); +free_percpu (tp_event->perf_events); tp_event->perf_events =NULL ;
@@ -156,7 +162,7 @@ __kprobes void *perf_trace_buf_prepare(iif (*rctxp <0 )return NULL ; - raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ()); + raw_data =this_cpu_ptr (perf_trace_buf[*rctxp]);/* zero the dead bytes from align to not leak stack to user */ memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64));
kernel/trace/trace_kprobe.c +2 -2
@@ -1362,7 +1362,7 @@ static __kprobes void kprobe_perf_func(sfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }
@@ -1395,7 +1395,7 @@ static __kprobes void kretprobe_perf_funfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }
kernel/trace/trace_syscalls.c +2 -2
@@ -469,7 +469,7 @@ static void perf_syscall_enter(struct ptsyscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); - head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->enter_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }
@@ -545,7 +545,7 @@ static void perf_syscall_exit(struct pt_ rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); - head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->exit_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }
Re: [PATCH 02b/10] perf, trace: Fix probe unregister race
Quoted 7 lines · Peter Zijlstra
tracepoint_probe_unregister()does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.Also, use
this_cpu_ptr()where possible.Signed-off-by Peter Zijlstra <[email protected]>
(and you previous patches too)
Quoted 101 lines
--- include/trace/ftrace.h | 2 +- kernel/trace/trace_event_perf.c | 10 ++++++++-- kernel/trace/trace_kprobe.c | 4 ++-- kernel/trace/trace_syscalls.c | 4 ++-- 4 files changed, 13 insertions(+), 7 deletions(-)Index:
linux-2.6/include/trace/ftrace.h
===================================================================include/trace/ftrace.h +1
-1@@ -791,7 +791,7 @@ perf_trace_templ_##call(struct ftrace_ev \ { assign; } \ \ - head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\ + head =this_cpu_ptr (event_call->perf_events); \perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ __count, __regs, head); \ }Index:
linux-2.6/kernel/trace/trace_event_perf.c
===================================================================kernel/trace/trace_event_perf.c +8
-2@@ -109,7 +109,7 @@ int perf_trace_enable(struct perf_eventif (WARN_ON_ONCE (!list))return -EINVAL ; - list =per_cpu_ptr (list,smp_processor_id ()); + list =this_cpu_ptr (list);hlist_add_head_rcu (&p_event->hlist_entry, list);return 0 ;@@ -130,6 +130,12 @@ void perf_trace_destroy(struct perf_even tp_event->perf_event_disable (tp_event); +/* +* Ensure our callback won't be called anymore. See +* tracepoint_probe_unregister() and __DO_TRACE(). +*/ +synchronize_sched (); +free_percpu (tp_event->perf_events); tp_event->perf_events =NULL ;@@ -156,7 +162,7 @@ __kprobes void *perf_trace_buf_prepare(iif (*rctxp <0 )return NULL ; - raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ()); + raw_data =this_cpu_ptr (perf_trace_buf[*rctxp]);/* zero the dead bytes from align to not leak stack to user */ memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64));Index:
linux-2.6/kernel/trace/trace_kprobe.c
===================================================================kernel/trace/trace_kprobe.c +2
-2@@ -1362,7 +1362,7 @@ static __kprobes void kprobe_perf_func(sfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }@@ -1395,7 +1395,7 @@ static __kprobes void kretprobe_perf_funfor (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }Index:
linux-2.6/kernel/trace/trace_syscalls.c
===================================================================kernel/trace/trace_syscalls.c +2
-2@@ -469,7 +469,7 @@ static void perf_syscall_enter(struct ptsyscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); - head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->enter_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }@@ -545,7 +545,7 @@ static void perf_syscall_exit(struct pt_ rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); - head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->exit_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }
Re: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Ingo MolnarQuoted 9 lines · Ingo Molnar
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line
What exactly is the puzzlement about?
The fact we use the hlist API not for hlist purpose but for a list.
Quoted 6 lines
Quoted 3 lines
I might miss the confusion, but hlist _are_ lists. Its just that their structure is slightly different that the regular struct
list_headstuff.Using an API in such a mixed way may cause puzzlement ;-)
I take that back - it's clear from the function prototype that we are using a mixed API: (hlist,list).
Sorry,
Ingo
Steven Rostedt
maintainerRe: [PATCH 09/10] perf: Remove more fastpath code
In reply to Peter Zijlstrakernel/perf_event.c again: · line 2979
Quoted 12 lines · Peter Zijlstra
plain text document attachment (perf-opt-output-more.patch) Sanity checks cost instructions
Signed-off-by Peter Zijlstra <[email protected]>---
kernel/perf_event.c +0
-7- -/* -* Check we didn't copy past our reservation window, taking the -* possible unsigned int wrap into account. -*/ -if (WARN_ON_ONCE (((long )(handle->head - handle->offset)) <0 )) -return ;
Are you sure you want to remove this? I mean, sanity checks are a small cost (I have several in the ftrace ring buffer) and can catch bugs in case something happens during development. Especially when the code is under a lot of flux.
I even have a sanity check that Ingo asked me to add, which would detect if a tracer (not the ring buffer, but the user of the ring buffer) recursed on itself. I think that check detected one bug in the function tracer in the begging, but hasn't caught anything since.
-- Steve
Quoted 27 lines
kernel/perf_event.c +3
-7- do { - unsigned long size = min(handle->size, len); + unsigned long size = min_t(unsigned long, handle->size, len); memcpy(handle->addr, buf, size);@@ -3054,15 +3045,12 @@ int perf_output_begin(struct perf_output goto fail; }while (local_cmpxchg (&data->head, offset, head) != offset); - handle->offset = offset; - handle->head = head; -if (head -local_read (&data->wakeup) > data->watermark)local_add (data->watermark, &data->wakeup); - handle->page = handle->offset >> (PAGE_SHIFT +page_order (data)); + handle->page = offset >> (PAGE_SHIFT +page_order (data)); handle->page &= data->nr_pages -1 ; - handle->size = handle->offset & ((PAGE_SIZE <<page_order (data)) -1 ); + handle->size = offset & ((PAGE_SIZE <<page_order (data)) -1 ); handle->addr = data->data_pages[handle->page]; handle->addr += handle->size; handle->size = (PAGE_SIZE <<page_order (data)) - handle->size;
Peter Zijlstra
maintainerRe: [PATCH 09/10] perf: Remove more fastpath code
Quoted 7 lines · Steven Rostedt
kernel/perf_event.c again: · line 2979
Quoted 2 lines
kernel/perf_event.c +0
-2-if (WARN_ON_ONCE (((long )(handle->head - handle->offset)) <0 )) -return ;Are you sure you want to remove this? I mean, sanity checks are a small cost (I have several in the ftrace ring buffer) and can catch bugs in case something happens during development. Especially when the code is under a lot of flux.
Yeah, I pondered adding a CONFIG_PERF_DEBUG for that.. thing is I was tracking 2 variables just for this, and that seemed a bit excessive.
Quoted 4 lines
I even have a sanity check that Ingo asked me to add, which would detect if a tracer (not the ring buffer, but the user of the ring buffer) recursed on itself. I think that check detected one bug in the function tracer in the begging, but hasn't caught anything since.
I still have recursion checks.
[tip:perf/core] perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraCommit-ID: 1c024eca51fdc965290acf342ae16a476c2189d0 Gitweb: http://git.kernel.org/tip/1c024eca51fdc965290acf342ae16a476c2189d0 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 May 2010 14:02:22 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:56 +0200
perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hlist to track events
Avoid the swevent hash-table by using per-tracepoint hlists.
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
LKML-Reference: <[email protected]>
--- include/linux/ftrace_event.h | 16 +++--- include/linux/perf_event.h | 6 +- include/trace/ftrace.h | 4 +- kernel/perf_event.c | 94 +++++++++++++++-------------- kernel/trace/trace_event_perf.c | 127 ++++++++++++++++++++------------------ kernel/trace/trace_kprobe.c | 9 ++- kernel/trace/trace_syscalls.c | 11 ++- 7 files changed, 143 insertions(+), 124 deletions(-)
include/linux/ftrace_event.h +8 -8
@@ -133,7 +133,7 @@ struct ftrace_event_call {void *data;int perf_refcount; -void *perf_data; +struct hlist_head *perf_events;int (*perf_event_enable)(struct ftrace_event_call *);void (*perf_event_disable)(struct ftrace_event_call *); };
@@ -192,9 +192,11 @@ struct perf_event; DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); -extern int perf_trace_enable (int event_id,void *data); -extern void perf_trace_disable (int event_id); -extern int ftrace_profile_set_filter (struct perf_event *event,int event_id, +extern int perf_trace_init (struct perf_event *event); +extern void perf_trace_destroy (struct perf_event *event); +extern int perf_trace_enable (struct perf_event *event); +extern void perf_trace_disable (struct perf_event *event); +extern int ftrace_profile_set_filter (struct perf_event *event,int event_id,char *filter_str);extern void ftrace_profile_free_filter (struct perf_event *event); extern void *perf_trace_buf_prepare(int size, unsigned short type,
@@ -202,11 +204,9 @@ extern void *perf_trace_buf_prepare(int size, unsigned short type,static inline void perf_trace_buf_submit (void *raw_data,int size,int rctx,u64 addr, -u64 count,struct pt_regs *regs,void *event) +u64 count,struct pt_regs *regs,void *head) { -struct trace_entry *entry = raw_data; - -perf_tp_event (entry->type, addr, count, raw_data, size, regs, event); +perf_tp_event (addr, count, raw_data, size, regs, head);perf_swevent_put_recursion_context (rctx); }#endif
include/linux/perf_event.h +4 -2
@@ -727,6 +727,7 @@ struct perf_event {perf_overflow_handler_t overflow_handler;#ifdef CONFIG_EVENT_TRACING +struct ftrace_event_call *tp_event;struct event_filter *filter;#endif
@@ -992,8 +993,9 @@ static inline bool perf_paranoid_kernel(void) }extern void perf_event_init (void ); -extern void perf_tp_event (int event_id,u64 addr,u64 count,void *record, -int entry_size,struct pt_regs *regs,void *event); +extern void perf_tp_event (u64 addr,u64 count,void *record, +int entry_size,struct pt_regs *regs, +struct hlist_head *head);extern void perf_bp_event (struct perf_event *event,void *data);#ifndef perf_misc_flags
include/trace/ftrace.h +3 -1
@@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_raw_##call *entry; \ u64 __addr = 0, __count = 1; \ + struct hlist_head *head; \ int __entry_size; \ int __data_size; \ int rctx; \
@@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \ \ { assign; } \ \ + head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ - __count, __regs, event_call->perf_data); \ + __count, __regs, head); \ }#undef DEFINE_EVENT
kernel/perf_event.c +48 -46
173 lines not on this page · raw message
kernel/trace/trace_event_perf.c +67 -60
197 lines not on this page · raw message
kernel/trace/trace_kprobe.c +6 -3
@@ -1341,6 +1341,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);struct ftrace_event_call *call = &tp->call;struct kprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
@@ -1361,7 +1362,8 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,for (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ()); +perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }/* Kretprobe profile handler */
@@ -1371,6 +1373,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);struct ftrace_event_call *call = &tp->call;struct kretprobe_trace_entry_head *entry; +struct hlist_head *head;u8 *data;int size, __size, i;int rctx;
@@ -1392,8 +1395,8 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,for (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); -perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , - regs, call->perf_data); + head =per_cpu_ptr (call->perf_events,smp_processor_id ()); +perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }static int probe_perf_enable (struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c +7 -4
@@ -438,6 +438,7 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) {struct syscall_metadata *sys_data;struct syscall_trace_enter *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
@@ -467,8 +468,9 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) rec->nr = syscall_nr;syscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->enter_event->perf_data); + + head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ()); +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysenter_enable (struct ftrace_event_call *call)
@@ -510,6 +512,7 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) {struct syscall_metadata *sys_data;struct syscall_trace_exit *rec; +struct hlist_head *head;int syscall_nr;int rctx;int size;
@@ -542,8 +545,8 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); -perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, - sys_data->exit_event->perf_data); + head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ()); +perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }int perf_sysexit_enable (struct ftrace_event_call *call)
[tip:perf/core] perf: Ensure that IOC_OUTPUT isn't used to create multi-writer buffers
In reply to Peter ZijlstraCommit-ID: 0f139300c9057c16b5833a4636b715b104fe0baa Gitweb: http://git.kernel.org/tip/0f139300c9057c16b5833a4636b715b104fe0baa Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 14:35:15 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:57 +0200
perf: Ensure that IOC_OUTPUT isn't used to create multi-writer buffers
Since we want to ensure buffers only have a single writer, we must avoid creating one with multiple.
LKML-Reference: <[email protected]>
--- kernel/perf_event.c | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-)
kernel/perf_event.c +19 -0
@@ -4920,6 +4920,13 @@ static int perf_event_set_output(struct perf_event *event, int output_fd)int fput_needed =0 ;int ret = -EINVAL ; +/* +* Don't allow output of inherited per-task events. This would +* create performance issues due to cross cpu access. +*/ +if (event->cpu ==-1 && event->attr.inherit) +return -EINVAL ; +if (!output_fd) goto set;
@@ -4940,6 +4947,18 @@ static int perf_event_set_output(struct perf_event *event, int output_fd)if (event->data) goto out; +/* +* Don't allow cross-cpu buffers +*/ +if (output_event->cpu != event->cpu) + goto out; + +/* +* If its not a per-cpu buffer, it must be the same task. +*/ +if (output_event->cpu ==-1 && output_event->ctx != event->ctx) + goto out; +atomic_long_inc (&output_file->f_count); set:
[tip:perf/core] perf-record: Remove -M
In reply to Peter ZijlstraCommit-ID: 57adc51dce9102b6641269dd04f5b99aac83b820 Gitweb: http://git.kernel.org/tip/57adc51dce9102b6641269dd04f5b99aac83b820 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 14:39:55 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:57 +0200
perf-record: Remove -M
Since it is not allowed to create cross-cpu (or cross-task) buffers, this option is no longer valid.
LKML-Reference: <[email protected]>
--- tools/perf/builtin-record.c | 36 ++++++++++++------------------------ 1 files changed, 12 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +12 -24
@@ -61,8 +61,6 @@ static bool call_graph = false;static bool inherit_stat = false;static bool no_samples = false;static bool sample_address = false; -static bool multiplex = false; -static int multiplex_fd =-1 ;static long samples =0 ;static u64 bytes_written =0 ;
@@ -366,27 +364,19 @@ try_again:*/ if (group && group_fd ==-1 ) group_fd = fd[nr_cpu][counter][thread_index]; -if (multiplex && multiplex_fd ==-1 ) - multiplex_fd = fd[nr_cpu][counter][thread_index]; -if (multiplex && fd[nr_cpu][counter][thread_index] != multiplex_fd) { - - ret =ioctl (fd[nr_cpu][counter][thread_index],PERF_EVENT_IOC_SET_OUTPUT , multiplex_fd); -assert (ret !=-1 ); - }else { - event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].events =POLLIN ; - nr_poll++; - - mmap_array[nr_cpu][counter][thread_index].counter = counter; - mmap_array[nr_cpu][counter][thread_index].prev =0 ; - mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; - mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, -PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); -if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { -error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); -exit (-1 ); - } + event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; + event_array[nr_poll].events =POLLIN ; + nr_poll++; + + mmap_array[nr_cpu][counter][thread_index].counter = counter; + mmap_array[nr_cpu][counter][thread_index].prev =0 ; + mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; + mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, +PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); +if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { +error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); +exit (-1 ); }if (filter !=NULL ) {
@@ -820,8 +810,6 @@ static const struct option options[] = { "Sample addresses"),OPT_BOOLEAN ('n' ,"no-samples" , &no_samples,"don't sample" ), -OPT_BOOLEAN ('M' ,"multiplex" , &multiplex, -"multiplex counter output in a single channel" ), OPT_END() };
[tip:perf/core] perf-record: Share per-cpu buffers
In reply to Peter ZijlstraCommit-ID: 0e2e63dd608bf5844ffae7bf7d860de18a62724c Gitweb: http://git.kernel.org/tip/0e2e63dd608bf5844ffae7bf7d860de18a62724c Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 14:45:26 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:58 +0200
perf-record: Share per-cpu buffers
It seems a waste of space to create a buffer per event, share it per-cpu.
LKML-Reference: <[email protected]>
--- tools/perf/builtin-record.c | 52 +++++++++++++++++++++++------------------- 1 files changed, 28 insertions(+), 24 deletions(-)
tools/perf/builtin-record.c +28 -24
@@ -82,7 +82,7 @@ struct mmap_data {unsigned int prev; }; -static struct mmap_data *mmap_array[MAX_NR_CPUS ][MAX_COUNTERS ]; +static struct mmap_data mmap_array[MAX_NR_CPUS ];static unsigned long mmap_read_head (struct mmap_data *md) {
@@ -365,18 +365,29 @@ try_again:if (group && group_fd ==-1 ) group_fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; - event_array[nr_poll].events =POLLIN ; - nr_poll++; - - mmap_array[nr_cpu][counter][thread_index].counter = counter; - mmap_array[nr_cpu][counter][thread_index].prev =0 ; - mmap_array[nr_cpu][counter][thread_index].mask = mmap_pages*page_size -1 ; - mmap_array[nr_cpu][counter][thread_index].base =mmap (NULL , (mmap_pages+1 )*page_size, -PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); -if (mmap_array[nr_cpu][counter][thread_index].base ==MAP_FAILED ) { -error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); -exit (-1 ); +if (counter || thread_index) { + ret =ioctl (fd[nr_cpu][counter][thread_index], +PERF_EVENT_IOC_SET_OUTPUT , + fd[nr_cpu][0 ][0 ]); +if (ret) { +error ("failed to set output: %d (%s)\n" , errno, +strerror (errno)); +exit (-1 ); + } + }else { + mmap_array[nr_cpu].counter = counter; + mmap_array[nr_cpu].prev =0 ; + mmap_array[nr_cpu].mask = mmap_pages*page_size -1 ; + mmap_array[nr_cpu].base =mmap (NULL , (mmap_pages+1 )*page_size, +PROT_READ |PROT_WRITE ,MAP_SHARED , fd[nr_cpu][counter][thread_index],0 ); +if (mmap_array[nr_cpu].base ==MAP_FAILED ) { +error ("failed to mmap with %d (%s)\n" , errno,strerror (errno)); +exit (-1 ); + } + + event_array[nr_poll].fd = fd[nr_cpu][counter][thread_index]; + event_array[nr_poll].events =POLLIN ; + nr_poll++; }if (filter !=NULL ) {
@@ -477,16 +488,11 @@ static struct perf_event_header finished_round_event = {static void mmap_read_all (void ) { -int i, counter, thread; +int i;for (i =0 ; i < nr_cpu; i++) { -for (counter =0 ; counter < nr_counters; counter++) { -for (thread =0 ; thread < thread_num; thread++) { -if (mmap_array[i][counter][thread].base) -mmap_read (&mmap_array[i][counter][thread]); - } - - } +if (mmap_array[i].base) +mmap_read (&mmap_array[i]); }if (perf_header__has_feat (&session->header,HEADER_TRACE_INFO ))
@@ -861,9 +867,7 @@ int cmd_record(int argc, const char **argv, const char *prefix __used)for (i =0 ; i <MAX_NR_CPUS ; i++) {for (j =0 ; j <MAX_COUNTERS ; j++) { fd[i][j] =malloc (sizeof (int )*thread_num); - mmap_array[i][j] =zalloc ( -sizeof (struct mmap_data )*thread_num); -if (!fd[i][j] || !mmap_array[i][j]) +if (!fd[i][j])return -ENOMEM ; } }
[tip:perf/core] perf: Fix wakeup storm for RO mmap()s
In reply to Peter ZijlstraCommit-ID: adb8e118f288dc4c569ac9a89010b81a4745fbf0 Gitweb: http://git.kernel.org/tip/adb8e118f288dc4c569ac9a89010b81a4745fbf0 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 16:21:55 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:58 +0200
perf: Fix wakeup storm for RO mmap()s
RO mmap()s don't update the tail pointer, so comparing against it for determining the written data size doesn't really do any good.
Keep track of when we last did a wakeup, and compare against that.
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 2 +- kernel/perf_event.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
include/linux/perf_event.h +1 -1
@@ -599,7 +599,7 @@ struct perf_mmap_data {local_t head;/* write position */ local_t nest;/* nested writers */ local_t events;/* event limit */ -local_t wakeup;/* needs a wakeup */ +local_t wakeup;/* wakeup stamp */ local_t lost;/* nr records lost */ long watermark;/* wakeup watermark */
kernel/perf_event.c +2 -2
@@ -3056,8 +3056,8 @@ int perf_output_begin(struct perf_output_handle *handle, handle->offset = offset; handle->head = head; -if (head - tail > data->watermark) -local_inc (&data->wakeup); +if (head -local_read (&data->wakeup) > data->watermark) +local_add (data->watermark, &data->wakeup);if (have_lost) { lost_event.header.type =PERF_RECORD_LOST ;
[tip:perf/core] perf: Optimize perf_output_copy()
In reply to Peter ZijlstraCommit-ID: 5d967a8be636a4f301a8daad642bd1007299d9ec Gitweb: http://git.kernel.org/tip/5d967a8be636a4f301a8daad642bd1007299d9ec Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 16:46:39 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:59 +0200
perf: Optimize perf_output_copy()
Reduce the clutter in perf_output_copy() by keeping an interator in perf_output_handle.
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 3 ++ kernel/perf_event.c | 54 +++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 28 deletions(-)
include/linux/perf_event.h +3 -0
@@ -806,6 +806,9 @@ struct perf_output_handle {unsigned long head;unsigned long offset;unsigned long wakeup; +unsigned long size; +void *addr; +int page;int nmi;int sample; };
kernel/perf_event.c +26 -28
71 lines not on this page · raw message
[tip:perf/core] perf: Optimize the !vmalloc backed buffer
In reply to Peter ZijlstraCommit-ID: 3cafa9fbb5c1d564b7b8e7224f493effbf04ffee Gitweb: http://git.kernel.org/tip/3cafa9fbb5c1d564b7b8e7224f493effbf04ffee Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 19:07:56 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:59 +0200
perf: Optimize the !vmalloc backed buffer
Reduce code and data by using the knowledge that for !PERF_USE_VMALLOC data_order is always 0.
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 2 +- kernel/perf_event.c | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 16 deletions(-)
include/linux/perf_event.h +1 -1
@@ -588,8 +588,8 @@ struct perf_mmap_data {struct rcu_head rcu_head;#ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; +int page_order;/* allocation order */ #endif -int data_order;/* allocation order */ int nr_pages;/* nr of data pages */ int writable;/* are we writable */ int nr_locked;/* nr pages mlocked */
kernel/perf_event.c +26 -15
102 lines not on this page · raw message
[tip:perf/core] perf: Remove more code from the fastpath
In reply to Peter ZijlstraCommit-ID: a94ffaaf55552769af328eaca9260fe6291c66c7 Gitweb: http://git.kernel.org/tip/a94ffaaf55552769af328eaca9260fe6291c66c7 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 19:50:07 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:37:59 +0200
perf: Remove more code from the fastpath
Sanity checks cost instructions.
LKML-Reference: <[email protected]>
--- include/linux/perf_event.h | 2 -- kernel/perf_event.c | 20 ++++---------------- 2 files changed, 4 insertions(+), 18 deletions(-)
include/linux/perf_event.h +0 -2
@@ -803,8 +803,6 @@ struct perf_cpu_context {struct perf_output_handle {struct perf_event *event;struct perf_mmap_data *data; -unsigned long head; -unsigned long offset;unsigned long wakeup;unsigned long size;void *addr;
kernel/perf_event.c +4 -16
@@ -2967,20 +2967,11 @@ again:preempt_enable (); } -void perf_output_copy (struct perf_output_handle *handle, +__always_inline void perf_output_copy(struct perf_output_handle *handle,const void *buf,unsigned int len) { - handle->offset += len; - -/* -* Check we didn't copy past our reservation window, taking the -* possible unsigned int wrap into account. -*/ -if (WARN_ON_ONCE (((long )(handle->head - handle->offset)) <0 )) -return ; - do { - unsigned long size = min(handle->size, len); + unsigned long size = min_t(unsigned long, handle->size, len); memcpy(handle->addr, buf, size);
@@ -3055,15 +3046,12 @@ int perf_output_begin(struct perf_output_handle *handle, goto fail; }while (local_cmpxchg (&data->head, offset, head) != offset); - handle->offset = offset; - handle->head = head; -if (head -local_read (&data->wakeup) > data->watermark)local_add (data->watermark, &data->wakeup); - handle->page = handle->offset >> (PAGE_SHIFT +page_order (data)); + handle->page = offset >> (PAGE_SHIFT +page_order (data)); handle->page &= data->nr_pages -1 ; - handle->size = handle->offset & ((PAGE_SIZE <<page_order (data)) -1 ); + handle->size = offset & ((PAGE_SIZE <<page_order (data)) -1 ); handle->addr = data->data_pages[handle->page]; handle->addr += handle->size; handle->size = (PAGE_SIZE <<page_order (data)) - handle->size;
[tip:perf/core] perf: Optimize perf_tp_event_match()
In reply to Peter ZijlstraCommit-ID: 580d607cd666dfabfc1c7b0fb08c8ac690c7c87f Gitweb: http://git.kernel.org/tip/580d607cd666dfabfc1c7b0fb08c8ac690c7c87f Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 20 May 2010 20:54:31 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 21 May 2010 11:38:00 +0200
perf: Optimize perf_tp_event_match()
Since we know tracepoints come from kernel context, avoid conditionals that try and establish that very fact.
LKML-Reference: <[email protected]>
--- kernel/perf_event.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-)
kernel/perf_event.c +4 -1
@@ -4496,7 +4496,10 @@ static int perf_tp_event_match(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { - if (perf_exclude_event(event, regs)) +/* +* All tracepoints are from kernel-space. +*/ +if (event->attr.exclude_kernel)return 0 ;if (!perf_tp_filter_match (event, data))
Steven Rostedt
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
In reply to Peter ZijlstraQuoted 13 lines · Peter Zijlstra
plain text document attachment (perf-trace-per-cpu-fold.patch) Avoid the swevent hash-table by using per-tracepoint hlists.
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
kernel/trace/trace_event_perf.c +3
-2-void perf_trace_disable (int event_id) +void perf_trace_destroy (struct perf_event *p_event) { -struct ftrace_event_call *event; +struct ftrace_event_call *tp_event = p_event->tp_event; +int i;
kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 130
Quoted 10 lines
kernel/trace/trace_event_perf.c +3
-6-mutex_lock (&event_mutex); -list_for_each_entry (event, &ftrace_events, list) { -if (event->id == event_id) { -perf_trace_event_disable (event); -module_put (event->mod); -break ; +if (--tp_event->perf_refcount >0 ) +return ; +
You remove the event_mutex and then update the tp_event->perf_refcount without any protection.
That tp_event->perf_refcount is global to the event not to the perf event.
-- Steve
Quoted 14 lines
kernel/trace/trace_event_perf.c +9
-1+ tp_event->perf_event_disable (tp_event); + +free_percpu (tp_event->perf_events); + tp_event->perf_events =NULL ; + +if (!--total_ref_count) { +for (i =0 ; i <4 ; i++) { +free_percpu (perf_trace_buf[i]); + perf_trace_buf[i] =NULL ; } } -mutex_unlock (&event_mutex); }
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 28 lines · Steven Rostedt
Quoted 13 lines · Peter Zijlstra
plain text document attachment (perf-trace-per-cpu-fold.patch) Avoid the swevent hash-table by using per-tracepoint hlists.
Also, avoid conditionals on the fast path by ordering with probe unregister so that we should never get on the callback path without the data being there.
kernel/trace/trace_event_perf.c +3
-2-void perf_trace_disable (int event_id) +void perf_trace_destroy (struct perf_event *p_event) { -struct ftrace_event_call *event; +struct ftrace_event_call *tp_event = p_event->tp_event; +int i;kernel/trace/trace_event_perf.c int perf_trace_enable(int event_id, void · line 130
Quoted 10 lines
kernel/trace/trace_event_perf.c +3
-6-mutex_lock (&event_mutex); -list_for_each_entry (event, &ftrace_events, list) { -if (event->id == event_id) { -perf_trace_event_disable (event); -module_put (event->mod); -break ; +if (--tp_event->perf_refcount >0 ) +return ; +You remove the
event_mutexand then update thetp_event->perf_refcountwithout any protection.
Quoted 2 lines
That
tp_event->perf_refcountis global to the event not to the perf event.
Oh, bugger, yes. I seem to have lost it on destroy..
Thanks!
Peter Zijlstra
maintainerRe: [PATCH 02/10] perf, trace: Use per-tracepoint-per-cpu hlist to track events
Quoted 5 lines · Peter Zijlstra
Quoted 3 lines
That
tp_event->perf_refcountis global to the event not to the perf event.Oh, bugger, yes. I seem to have lost it on destroy..
---
Subject: perf, trace: Fix perf_trace_destroy(), mutex went missing
From: Peter Zijlstra <[email protected]>
Date: Fri May 21 16:22:33 CEST 2010
Steve spotted I forgot to do the destroy under event_mutex.
LKML-Reference: <new-submission>
--- kernel/trace/trace_event_perf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
kernel/trace/trace_event_perf.c +4 -1
@@ -125,8 +125,9 @@ void perf_trace_destroy(struct perf_evenstruct ftrace_event_call *tp_event = p_event->tp_event;int i; +mutex_lock (&event_mutex);if (--tp_event->perf_refcount >0 ) -return ; + goto out; tp_event->perf_event_disable (tp_event);
@@ -145,6 +146,8 @@ void perf_trace_destroy(struct perf_even perf_trace_buf[i] =NULL ; } } +out: +mutex_unlock (&event_mutex); } __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
In reply to Peter ZijlstraQuoted 7 lines · Peter Zijlstra
[...]
include/trace/ftrace.h +0
-2@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev static notrace void perf_trace_##call(proto) \ { \ struct ftrace_event_call *event_call = &event_##call; \ - struct pt_regs *__regs = &get_cpu_var(perf_trace_regs); \ - \
include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 806
Quoted 10 lines
include/trace/ftrace.h +3
-4-perf_fetch_caller_regs (__regs,1 ); \ - \ - perf_trace_templ_##template(event_call, __regs, args); \ +struct pt_regs __regs; \ \ - put_cpu_var(perf_trace_regs); \ +perf_fetch_caller_regs (&__regs,1 ); \ + perf_trace_templ_##template(event_call, &__regs, args); \ }
To what extent are you worried about something the size of struct pt_regs being auto/stack allocated?
- FChE
Steven Rostedt
maintainerRe: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
Quoted 14 lines · Frank Ch. Eigler
Quoted 12 lines · Peter Zijlstra
[...]
include/trace/ftrace.h +1
-5@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_ev static notrace void perf_trace_##call(proto) \ { \ struct ftrace_event_call *event_call = &event_##call; \ - struct pt_regs *__regs = &get_cpu_var(perf_trace_regs); \ - \ - perf_fetch_caller_regs(__regs, 1); \ - \ - perf_trace_templ_##template(event_call, __regs, args); \ + struct pt_regs __regs; \ \
Quoted 8 lines
include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 806
Quoted 5 lines
include/trace/ftrace.h +2
-1-put_cpu_var (perf_trace_regs); \ +perf_fetch_caller_regs (&__regs,1 ); \ + perf_trace_templ_##template(event_call, &__regs, args); \ }To what extent are you worried about something the size of struct
pt_regsbeing auto/stack allocated?
Isn't pt_regs already allocated on the stack when interrupted?
-- Steve
Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
Hi -
Quoted 10 lines · Steven Rostedt
[...]
Quoted 7 lines
include/trace/ftrace.h perf_trace_templ_##call(struct ftrace_ev · line 802
Quoted 4 lines
include/trace/ftrace.h +1
-2-struct pt_regs *__regs = &get_cpu_var (perf_trace_regs); \ - [...] +struct pt_regs __regs; \To what extent are you worried about something the size of struct
pt_regsbeing auto/stack allocated?Isn't
pt_regsalready allocated on the stack when interrupted?
Maybe, but here we're about to ask for another multi-hundred-byte one. Does your stack-frame-size ftrace widget gripe about this change?
- FChE
Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
In reply to Peter Zijlstrakernel/trace/trace_event_perf.c void perf_trace_disable(int event_id) · line 155
Quoted 4 lines · Peter Zijlstra
kernel/trace/trace_event_perf.c +1
-1entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, *irq_flags, pc); +tracing_generic_entry_update (entry, regs->flags, pc); entry->type = type;
This is breaking on powerpc -- we don't have a "flags" element in struct pt_regs. What is it trying to do? Get the interrupt enable state, as local_irq_save(flags) would provide?
Paul.
Peter Zijlstra
maintainerRe: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
Quoted 10 lines · Paul Mackerras
Review
Quoted 4 lines · Peter Zijlstra
(quoted diff) +1
-1entry = (struct trace_entry *)raw_data; - tracing_generic_entry_update(entry, *irq_flags, pc); + tracing_generic_entry_update(entry, regs->flags, pc); entry->type = type;This is breaking on powerpc -- we don't have a "flags" element in struct
pt_regs. What is it trying to do? Get the interrupt enable state, aslocal_irq_save(flags)would provide?
Yes, I fouled that up.
On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().
So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.
Possibly we could remove the local_save_flags() from the perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags().
Something like:
--- arch/x86/kernel/cpu/perf_event.c | 1 - kernel/trace/trace_event_perf.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
@@ -1717,7 +1717,6 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski*/ regs->bp =rewind_frame_pointer (skip +1 ); regs->cs = __KERNEL_CS; -local_save_flags (regs->flags); }unsigned long perf_instruction_pointer (struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, struct pt_regs *regs, int *rctxp) { struct trace_entry *entry; +unsigned long flags;char *raw_data;int pc;
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64)); entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, regs->flags, pc); +local_save_flags (flags); +tracing_generic_entry_update (entry, flags, pc); entry->type = type;return raw_data;
Paul Mackerras
Acked-byRe: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
Quoted 13 lines · Peter Zijlstra
Quoted 3 lines · Paul Mackerras
This is breaking on powerpc -- we don't have a "flags" element in struct
pt_regs. What is it trying to do? Get the interrupt enable state, aslocal_irq_save(flags)would provide?Yes, I fouled that up.
On x86
perf_arch_fetch_caller_regs()storespt_regs::flags, which is the same as used inlocal_save_flags().So to avoid another
local_save_flags()for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.
Quoted 6 lines
Possibly we could remove the
local_save_flags()from theperf_arch_fetch_caller_regs()as I'm not sure we actually usept_regs::flags(Frederic?) and leave the ftrace thing to uselocal_save_flags().Something like:
Yes, that works -- at least, it compiles and seems to run. :)
Paul.
Re: [PATCH 01/10] perf, trace: Remove IRQ-disable from perf/tracepoint interaction
In reply to Peter ZijlstraQuoted 17 lines · Peter Zijlstra
Quoted 10 lines · Paul Mackerras
kernel/trace/trace_event_perf.c __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, · line 180
Quoted 4 lines · Peter Zijlstra
kernel/trace/trace_event_perf.c +1
-1entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, *irq_flags, pc); +tracing_generic_entry_update (entry, regs->flags, pc); entry->type = type;This is breaking on powerpc -- we don't have a "flags" element in struct
pt_regs. What is it trying to do? Get the interrupt enable state, aslocal_irq_save(flags)would provide?Yes, I fouled that up.
On x86
perf_arch_fetch_caller_regs()storespt_regs::flags, which is the same as used inlocal_save_flags().
Quoted 7 lines
So to avoid another
local_save_flags()for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.Possibly we could remove the
local_save_flags()from theperf_arch_fetch_caller_regs()as I'm not sure we actually usept_regs::flags(Frederic?) and leave the ftrace thing to uselocal_save_flags().
It was an anticipation for further uses. But since further are not appearing, you can safely remove it.
Thanks.
Peter Zijlstra
maintainerPATCH 1/1 perf, trace: Fix !x86 build issue
In reply to Peter ZijlstraSubject: perf, trace: Fix !x86 build issue
From: Peter Zijlstra <[email protected]>
Date: Tue May 25 09:22:38 CEST 2010
Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.
The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.
Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.
--- arch/x86/kernel/cpu/perf_event.c | 6 +++++- kernel/trace/trace_event_perf.c | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct*/ regs->bp =rewind_frame_pointer (skip +1 ); regs->cs = __KERNEL_CS; -local_save_flags (regs->flags); +/* +* We abuse bit 3 to pass exact information, see perf_misc_flags +* and the comment with PERF_EFLAGS_EXACT. +*/ + regs->flags =0 ; }unsigned long perf_instruction_pointer (struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(i struct pt_regs *regs, int *rctxp) { struct trace_entry *entry; +unsigned long flags;char *raw_data;int pc;
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(imemset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64)); entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, regs->flags, pc); +local_save_flags (flags); +tracing_generic_entry_update (entry, flags, pc); entry->type = type;return raw_data;
tip-bot for Peter Zijlstra
maintainer[tip:perf/core] perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction
In reply to Paul MackerrasCommit-ID: 23ef672d1e8236e6fe606a39963ba2d576972d84 Gitweb: http://git.kernel.org/tip/23ef672d1e8236e6fe606a39963ba2d576972d84 Author: Peter Zijlstra <[email protected]> AuthorDate: Sun, 23 May 2010 20:16:27 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 25 May 2010 09:02:08 +0200
perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction
On x86 perf_arch_fetch_caller_regs() stores pt_regs::flags, which is the same as used in local_save_flags().
So to avoid another local_save_flags() for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.
[ Possibly we could remove the local_save_flags() from the
perf_arch_fetch_caller_regs() as I'm not sure we actually use pt_regs::flags (Frederic?) and leave the ftrace thing to use local_save_flags(). ]
LKML-Reference: <20100524042958.GA10628@drongo>
--- arch/x86/kernel/cpu/perf_event.c | 1 - kernel/trace/trace_event_perf.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
@@ -1717,7 +1717,6 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski*/ regs->bp =rewind_frame_pointer (skip +1 ); regs->cs = __KERNEL_CS; -local_save_flags (regs->flags); }unsigned long perf_instruction_pointer (struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, struct pt_regs *regs, int *rctxp) { struct trace_entry *entry; +unsigned long flags;char *raw_data;int pc;
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64)); entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, regs->flags, pc); +local_save_flags (flags); +tracing_generic_entry_update (entry, flags, pc); entry->type = type;return raw_data;
Re: [tip:perf/core] perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction
Quoted 20 lines · tip-bot for Peter Zijlstra
Commit-ID: 23ef672d1e8236e6fe606a39963ba2d576972d84 Gitweb: http://git.kernel.org/tip/23ef672d1e8236e6fe606a39963ba2d576972d84 Author: Peter Zijlstra <[email protected]> AuthorDate: Sun, 23 May 2010 20:16:27 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 25 May 2010 09:02:08 +0200perf, trace: Fix IRQ-disable removal from perf/tracepoint interaction
On x86
perf_arch_fetch_caller_regs()storespt_regs::flags, which is the same as used inlocal_save_flags().So to avoid another
local_save_flags()for the ftrace code, I wanted to reuse it, and totally forgot the !x86 side of things.[ Possibly we could remove the
local_save_flags()from the
perf_arch_fetch_caller_regs()as I'm not sure we actually usept_regs::flags(Frederic?) and leave the ftrace thing to uselocal_save_flags(). ]
Quoted 8 lines
Signed-off-by Peter Zijlstra <[email protected]>Acked-by Paul Mackerras <[email protected]>Cc Arnaldo Carvalho de Melo <[email protected]>Cc Frederic Weisbecker <[email protected]>Cc Mike Galbraith <[email protected]>Cc Steven Rostedt <[email protected]>LKML-Reference: <20100524042958.GA10628@drongo>
Signed-off-by Ingo Molnar <[email protected]>
Could you replace this with:
---
Subject: perf, trace: Fix !x86 build issue
From: Peter Zijlstra <[email protected]>
Date: Tue May 25 09:22:38 CEST 2010
Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.
The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.
Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.
--- arch/x86/kernel/cpu/perf_event.c | 6 +++++- kernel/trace/trace_event_perf.c | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct*/ regs->bp =rewind_frame_pointer (skip +1 ); regs->cs = __KERNEL_CS; -local_save_flags (regs->flags); +/* +* We abuse bit 3 to pass exact information, see perf_misc_flags +* and the comment with PERF_EFLAGS_EXACT. +*/ + regs->flags =0 ; }unsigned long perf_instruction_pointer (struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(i struct pt_regs *regs, int *rctxp) { struct trace_entry *entry; +unsigned long flags;char *raw_data;int pc;
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(imemset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64)); entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, regs->flags, pc); +local_save_flags (flags); +tracing_generic_entry_update (entry, flags, pc); entry->type = type;return raw_data;
tip-bot for Peter Zijlstra
maintainer[tip:perf/core] perf, trace: Fix !x86 build bug
Commit-ID: 87f44bbc246c5244c76a701f8eefba7788bce64a Gitweb: http://git.kernel.org/tip/87f44bbc246c5244c76a701f8eefba7788bce64a Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 25 May 2010 11:02:55 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 25 May 2010 11:28:49 +0200
perf, trace: Fix !x86 build bug
Patch b7e2ecef92 (perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction) made the unfortunate mistake of assuming the world is x86 only, correct this.
The problem was that perf_fetch_caller_regs() did local_save_flags() into regs->flags, and I re-used that to remove another local_save_flags(), forgetting !x86 doesn't have regs->flags.
Do the reverse, remove the local_save_flags() from perf_fetch_caller_regs() and let the ftrace site do the local_save_flags() instead.
LKML-Reference: <1274778175.5882.623.camel@twins>
--- arch/x86/kernel/cpu/perf_event.c | 6 +++++- kernel/trace/trace_event_perf.c | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event.c +5 -1
@@ -1717,7 +1717,11 @@ void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int ski*/ regs->bp =rewind_frame_pointer (skip +1 ); regs->cs = __KERNEL_CS; -local_save_flags (regs->flags); +/* +* We abuse bit 3 to pass exact information, see perf_misc_flags +* and the comment with PERF_EFLAGS_EXACT. +*/ + regs->flags =0 ; }unsigned long perf_instruction_pointer (struct pt_regs *regs)
kernel/trace/trace_event_perf.c +3 -1
@@ -157,6 +157,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, struct pt_regs *regs, int *rctxp) { struct trace_entry *entry; +unsigned long flags;char *raw_data;int pc;
@@ -174,7 +175,8 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64)); entry = (struct trace_entry *)raw_data; -tracing_generic_entry_update (entry, regs->flags, pc); +local_save_flags (flags); +tracing_generic_entry_update (entry, flags, pc); entry->type = type;return raw_data;
[tip:perf/urgent] perf_events, trace: Fix probe unregister race
In reply to Peter ZijlstraCommit-ID: 3771f0771154675d4a0ca780be2411f3cc357208 Gitweb: http://git.kernel.org/tip/3771f0771154675d4a0ca780be2411f3cc357208 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 21 May 2010 12:31:09 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 31 May 2010 08:46:09 +0200
perf_events, trace: Fix probe unregister race
tracepoint_probe_unregister() does not synchronize against the probe callbacks, so do that explicitly. This properly serializes the callbacks and the free of the data used therein.
Also, use this_cpu_ptr() where possible.
LKML-Reference: <1274438476.1674.1702.camel@laptop>
--- include/trace/ftrace.h | 2 +- kernel/trace/trace_event_perf.c | 10 ++++++++-- kernel/trace/trace_kprobe.c | 4 ++-- kernel/trace/trace_syscalls.c | 4 ++-- 4 files changed, 13 insertions(+), 7 deletions(-)
include/trace/ftrace.h +1 -1
@@ -725,7 +725,7 @@ perf_trace_##call(void *__data, proto) \ \ { assign; } \ \ - head =per_cpu_ptr (event_call->perf_events,smp_processor_id ());\ + head =this_cpu_ptr (event_call->perf_events); \perf_trace_buf_submit (entry, __entry_size, rctx, __addr, \ __count, &__regs, head); \ }
kernel/trace/trace_event_perf.c +8 -2
@@ -116,7 +116,7 @@ int perf_trace_enable(struct perf_event *p_event)if (WARN_ON_ONCE (!list))return -EINVAL ; - list =per_cpu_ptr (list,smp_processor_id ()); + list =this_cpu_ptr (list);hlist_add_head_rcu (&p_event->hlist_entry, list);return 0 ;
@@ -142,6 +142,12 @@ void perf_trace_destroy(struct perf_event *p_event) tp_event->class->perf_probe, tp_event); +/* +* Ensure our callback won't be called anymore. See +* tracepoint_probe_unregister() and __DO_TRACE(). +*/ +synchronize_sched (); +free_percpu (tp_event->perf_events); tp_event->perf_events =NULL ;
@@ -169,7 +175,7 @@ __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,if (*rctxp <0 )return NULL ; - raw_data =per_cpu_ptr (perf_trace_buf[*rctxp],smp_processor_id ()); + raw_data =this_cpu_ptr (perf_trace_buf[*rctxp]);/* zero the dead bytes from align to not leak stack to user */ memset (&raw_data[size -sizeof (u64)],0 ,sizeof (u64));
kernel/trace/trace_kprobe.c +2 -2
@@ -1359,7 +1359,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,for (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ip,1 , regs, head); }
@@ -1392,7 +1392,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,for (i =0 ; i < tp->nr_args; i++)call_fetch (&tp->args[i].fetch, regs, data + tp->args[i].offset); - head =per_cpu_ptr (call->perf_events,smp_processor_id ()); + head =this_cpu_ptr (call->perf_events);perf_trace_buf_submit (entry, size, rctx, entry->ret_ip,1 , regs, head); }
kernel/trace/trace_syscalls.c +2 -2
@@ -519,7 +519,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)syscall_get_arguments (current, regs,0 , sys_data->nb_args, (unsigned long *)&rec->args); - head =per_cpu_ptr (sys_data->enter_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->enter_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }
@@ -595,7 +595,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) rec->nr = syscall_nr; rec->ret =syscall_get_return_value (current, regs); - head =per_cpu_ptr (sys_data->exit_event->perf_events,smp_processor_id ()); + head =this_cpu_ptr (sys_data->exit_event->perf_events);perf_trace_buf_submit (rec, size, rctx,0 ,1 , regs, head); }
[tip:perf/urgent] perf_events, trace: Fix perf_trace_destroy(), mutex went missing
In reply to Peter ZijlstraCommit-ID: 2e97942fe57864588774f173cf4cd7bb68968b76 Gitweb: http://git.kernel.org/tip/2e97942fe57864588774f173cf4cd7bb68968b76 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 21 May 2010 16:22:33 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 31 May 2010 08:46:09 +0200
perf_events, trace: Fix perf_trace_destroy(), mutex went missing
Steve spotted I forgot to do the destroy under event_mutex.
LKML-Reference: <1274451913.1674.1707.camel@laptop>
--- kernel/trace/trace_event_perf.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-)
kernel/trace/trace_event_perf.c +4 -1
@@ -132,8 +132,9 @@ void perf_trace_destroy(struct perf_event *p_event)struct ftrace_event_call *tp_event = p_event->tp_event;int i; +mutex_lock (&event_mutex);if (--tp_event->perf_refcount >0 ) -return ; + goto out;if (tp_event->class->reg) tp_event->class->reg (tp_event,TRACE_REG_PERF_UNREGISTER );
@@ -157,6 +158,8 @@ void perf_trace_destroy(struct perf_event *p_event) perf_trace_buf[i] =NULL ; } } +out: +mutex_unlock (&event_mutex); } __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,