perf: cleanup and fixes
Subsystems (7)
cover perf: cleanup and fixes
cleanups and fixes for perf core, although the last one could go to perf/urgent.
It has only been tested in x86.
Frederic Weisbecker (6):
perf: Drop unappropriate tests on arch callchains
perf: Generalize callchain_store()
perf: Generalize some arch callchain code
perf: Factorize callchain context handling
perf: Fix race in callchains
perf: Fix double put_ctx
arch/arm/kernel/perf_event.c | 62 +-------- arch/powerpc/kernel/perf_callchain.c | 86 ++++--------- arch/sh/kernel/perf_callchain.c | 50 +------- arch/sparc/kernel/perf_event.c | 69 ++++------- arch/x86/kernel/cpu/perf_event.c | 82 +++---------- include/linux/perf_event.h | 16 +++- kernel/perf_event.c | 228 +++++++++++++++++++++++++++++----- 7 files changed, 289 insertions(+), 304 deletions(-)
RFC PATCH 1/6 perf: Drop unappropriate tests on arch callchains
Drop the TASK_RUNNING test on user tasks for callchains as this check doesn't seem to make any sense.
Also remove the tests for !current that is not supposed to happen and current->pid as this should be handled at the generic level, with exclude_idle attribute.
--- arch/arm/kernel/perf_event.c | 6 ------ arch/sh/kernel/perf_callchain.c | 3 --- arch/x86/kernel/cpu/perf_event.c | 3 --- 3 files changed, 0 insertions(+), 12 deletions(-)
arch/arm/kernel/perf_event.c +0 -6
@@ -3107,12 +3107,6 @@ perf_do_callchain(struct pt_regs *regs, is_user =user_mode (regs); -if (!current || !current->pid) -return ; - -if (is_user && current->state !=TASK_RUNNING ) -return ; -if (!is_user)perf_callchain_kernel (regs, entry);
arch/sh/kernel/perf_callchain.c +0 -3
@@ -68,9 +68,6 @@ perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry) is_user =user_mode (regs); -if (is_user && current->state !=TASK_RUNNING ) -return ; -/* * Only the kernel side is implemented for now. */
arch/x86/kernel/cpu/perf_event.c +0 -3
@@ -1703,9 +1703,6 @@ perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry) is_user =user_mode (regs); -if (is_user && current->state !=TASK_RUNNING ) -return ; -if (!is_user)perf_callchain_kernel (regs, entry);
Signature · 1 line
1.6.2.3
RFC PATCH 2/6 perf: Generalize callchain_store()
In reply to Frederic Weisbeckercallchain_store() is the same on every archs, inline it in perf_event.h and rename it to perf_callchain_store() to avoid any collision.
This removes repetitive code.
--- arch/arm/kernel/perf_event.c | 15 +++--------- arch/powerpc/kernel/perf_callchain.c | 40 ++++++++++++---------------------- arch/sh/kernel/perf_callchain.c | 11 ++------ arch/sparc/kernel/perf_event.c | 26 ++++++++------------- arch/x86/kernel/cpu/perf_event.c | 20 ++++++----------- include/linux/perf_event.h | 7 ++++++ 6 files changed, 45 insertions(+), 74 deletions(-)
arch/arm/kernel/perf_event.c +4 -11
@@ -3001,13 +3001,6 @@ arch_initcall(init_hw_perf_events);/* * Callchain handling code. */ -static inline void -callchain_store (struct perf_callchain_entry *entry, -u64 ip) -{ -if (entry->nr <PERF_MAX_STACK_DEPTH ) - entry->ip[entry->nr++] = ip; -}/* * The registers we're interested in are at the end of the variable
@@ -3039,7 +3032,7 @@ user_backtrace(struct frame_tail *tail,if (__copy_from_user_inatomic (&buftail, tail,sizeof (buftail)))return NULL ; -callchain_store (entry, buftail.lr); +perf_callchain_store (entry, buftail.lr);/* * Frame pointers should strictly progress back up the stack
@@ -3057,7 +3050,7 @@ perf_callchain_user(struct pt_regs *regs, {struct frame_tail *tail; -callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry,PERF_CONTEXT_USER );if (!user_mode (regs)) regs =task_pt_regs (current);
@@ -3078,7 +3071,7 @@ callchain_trace(struct stackframe *fr, void *data) {struct perf_callchain_entry *entry = data; -callchain_store (entry, fr->pc); +perf_callchain_store (entry, fr->pc);return 0 ; }
@@ -3088,7 +3081,7 @@ perf_callchain_kernel(struct pt_regs *regs, {struct stackframe fr; -callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); fr.fp = regs->ARM_fp; fr.sp = regs->ARM_sp; fr.lr = regs->ARM_lr;
arch/powerpc/kernel/perf_callchain.c +14 -26
@@ -23,18 +23,6 @@#include "ppc32.h" #endif -/* -* Store another value in a callchain_entry. -*/ -static inline void callchain_store (struct perf_callchain_entry *entry,u64 ip) -{ -unsigned int nr = entry->nr; - -if (nr <PERF_MAX_STACK_DEPTH ) { - entry->ip[nr] = ip; - entry->nr = nr +1 ; - } -}/* * Is sp valid as the address of the next kernel stack frame after prev_sp?
@@ -69,8 +57,8 @@ static void perf_callchain_kernel(struct pt_regs *regs, lr = regs->link; sp = regs->gpr[1 ]; -callchain_store (entry,PERF_CONTEXT_KERNEL ); -callchain_store (entry, regs->nip); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry, regs->nip);if (!validate_sp (sp, current,STACK_FRAME_OVERHEAD ))return ;
@@ -89,7 +77,7 @@ static void perf_callchain_kernel(struct pt_regs *regs, next_ip = regs->nip; lr = regs->link; level =0 ; -callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); }else {if (level ==0 )
@@ -111,7 +99,7 @@ static void perf_callchain_kernel(struct pt_regs *regs, ++level; } -callchain_store (entry, next_ip); +perf_callchain_store (entry, next_ip);if (!valid_next_sp (next_sp, sp))return ; sp = next_sp;
@@ -246,8 +234,8 @@ static void perf_callchain_user_64(struct pt_regs *regs, next_ip = regs->nip; lr = regs->link; sp = regs->gpr[1 ]; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, next_ip); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, next_ip);for (;;) { fp = (unsigned long __user *) sp;
@@ -276,14 +264,14 @@ static void perf_callchain_user_64(struct pt_regs *regs, read_user_stack_64(&uregs[PT_R1], &sp))return ; level =0 ; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, next_ip); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, next_ip);continue ; }if (level ==0 ) next_ip = lr; -callchain_store (entry, next_ip); +perf_callchain_store (entry, next_ip); ++level; sp = next_sp; }
@@ -447,8 +435,8 @@ static void perf_callchain_user_32(struct pt_regs *regs, next_ip = regs->nip; lr = regs->link; sp = regs->gpr[1 ]; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, next_ip); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, next_ip);while (entry->nr <PERF_MAX_STACK_DEPTH ) { fp = (unsigned int __user *) (unsigned long ) sp;
@@ -470,14 +458,14 @@ static void perf_callchain_user_32(struct pt_regs *regs, read_user_stack_32(&uregs[PT_R1], &sp))return ; level =0 ; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, next_ip); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, next_ip);continue ; }if (level ==0 ) next_ip = lr; -callchain_store (entry, next_ip); +perf_callchain_store (entry, next_ip); ++level; sp = next_sp; }
arch/sh/kernel/perf_callchain.c +3 -8
@@ -14,11 +14,6 @@#include <asm/unwinder.h> #include <asm/ptrace.h> -static inline void callchain_store (struct perf_callchain_entry *entry,u64 ip) -{ -if (entry->nr <PERF_MAX_STACK_DEPTH ) - entry->ip[entry->nr++] = ip; -}static void callchain_warning (void *data,char *msg) {
@@ -39,7 +34,7 @@ static void callchain_address(void *data, unsigned long addr, int reliable)struct perf_callchain_entry *entry = data;if (reliable) -callchain_store (entry, addr); +perf_callchain_store (entry, addr); }static const struct stacktrace_ops callchain_ops = {
@@ -52,8 +47,8 @@ static const struct stacktrace_ops callchain_ops = {static void perf_callchain_kernel (struct pt_regs *regs,struct perf_callchain_entry *entry) { -callchain_store (entry,PERF_CONTEXT_KERNEL ); -callchain_store (entry, regs->pc); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry, regs->pc);unwind_stack (NULL , regs,NULL , &callchain_ops, entry); }
arch/sparc/kernel/perf_event.c +10 -16
@@ -1282,12 +1282,6 @@ void __init init_hw_perf_events(void)register_die_notifier (&perf_event_nmi_notifier); } -static inline void callchain_store (struct perf_callchain_entry *entry,u64 ip) -{ -if (entry->nr <PERF_MAX_STACK_DEPTH ) - entry->ip[entry->nr++] = ip; -} -static void perf_callchain_kernel (struct pt_regs *regs,struct perf_callchain_entry *entry) {
@@ -1296,8 +1290,8 @@ static void perf_callchain_kernel(struct pt_regs *regs,int graph =0 ;#endif -callchain_store (entry,PERF_CONTEXT_KERNEL ); -callchain_store (entry, regs->tpc); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry, regs->tpc); ksp = regs->u_regs[UREG_I6 ]; fp = ksp +STACK_BIAS ;
@@ -1321,13 +1315,13 @@ static void perf_callchain_kernel(struct pt_regs *regs, pc = sf->callers_pc; fp = (unsigned long )sf->fp +STACK_BIAS ; } -callchain_store (entry, pc); +perf_callchain_store (entry, pc);#ifdef CONFIG_FUNCTION_GRAPH_TRACER if ((pc +8UL ) == (unsigned long ) &return_to_handler) {int index = current->curr_ret_stack;if (current->ret_stack && index >= graph) { pc = current->ret_stack[index - graph].ret; -callchain_store (entry, pc); +perf_callchain_store (entry, pc); graph++; } }
@@ -1340,8 +1334,8 @@ static void perf_callchain_user_64(struct pt_regs *regs, {unsigned long ufp; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, regs->tpc); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, regs->tpc); ufp = regs->u_regs[UREG_I6 ] +STACK_BIAS ; do {
@@ -1354,7 +1348,7 @@ static void perf_callchain_user_64(struct pt_regs *regs, pc = sf.callers_pc; ufp = (unsigned long )sf.fp +STACK_BIAS ; -callchain_store (entry, pc); +perf_callchain_store (entry, pc); }while (entry->nr <PERF_MAX_STACK_DEPTH ); }
@@ -1363,8 +1357,8 @@ static void perf_callchain_user_32(struct pt_regs *regs, {unsigned long ufp; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, regs->tpc); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, regs->tpc); ufp = regs->u_regs[UREG_I6 ] &0xffffffffUL ; do {
@@ -1377,7 +1371,7 @@ static void perf_callchain_user_32(struct pt_regs *regs, pc = sf.callers_pc; ufp = (unsigned long )sf.fp; -callchain_store (entry, pc); +perf_callchain_store (entry, pc); }while (entry->nr <PERF_MAX_STACK_DEPTH ); }
arch/x86/kernel/cpu/perf_event.c +7 -13
@@ -1571,12 +1571,6 @@ const struct pmu *hw_perf_event_init(struct perf_event *event)* callchain support */ -static inline -void callchain_store (struct perf_callchain_entry *entry,u64 ip) -{ -if (entry->nr <PERF_MAX_STACK_DEPTH ) - entry->ip[entry->nr++] = ip; -} static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry); static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
@@ -1602,7 +1596,7 @@ static void backtrace_address(void *data, unsigned long addr, int reliable) {struct perf_callchain_entry *entry = data; -callchain_store (entry, addr); +perf_callchain_store (entry, addr); }static const struct stacktrace_ops backtrace_ops = {
@@ -1616,8 +1610,8 @@ static const struct stacktrace_ops backtrace_ops = {static void perf_callchain_kernel (struct pt_regs *regs,struct perf_callchain_entry *entry) { -callchain_store (entry,PERF_CONTEXT_KERNEL ); -callchain_store (entry, regs->ip); +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_store (entry, regs->ip);dump_trace (NULL , regs,NULL , regs->bp, &backtrace_ops, entry); }
@@ -1646,7 +1640,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)if (fp <compat_ptr (regs->sp))break ; -callchain_store (entry, frame.return_address); +perf_callchain_store (entry, frame.return_address); fp =compat_ptr (frame.next_frame); }return 1 ;
@@ -1670,8 +1664,8 @@ perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry) fp = (void __user *)regs->bp; -callchain_store (entry,PERF_CONTEXT_USER ); -callchain_store (entry, regs->ip); +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_store (entry, regs->ip);if (perf_callchain_user32 (regs, entry))return ;
@@ -1688,7 +1682,7 @@ perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)if ((unsigned long )fp < regs->sp)break ; -callchain_store (entry, frame.return_address); +perf_callchain_store (entry, frame.return_address); fp = frame.next_frame; } }
include/linux/perf_event.h +7 -0
@@ -978,6 +978,13 @@ extern void perf_event_fork(struct task_struct *tsk);extern struct perf_callchain_entry *perf_callchain (struct pt_regs *regs); +static inline void +perf_callchain_store (struct perf_callchain_entry *entry,u64 ip) +{ +if (entry->nr <PERF_MAX_STACK_DEPTH ) + entry->ip[entry->nr++] = ip; +} +extern int sysctl_perf_event_paranoid;extern int sysctl_perf_event_mlock;extern int sysctl_perf_event_sample_rate;
Signature · 1 line
1.6.2.3
RFC PATCH 3/6 perf: Generalize some arch callchain code
In reply to Frederic Weisbecker- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default perf_callchain_buffer() implementation that x86 overrides.
- Centralize all the kernel/user regs handling and invoke new arch handlers from there: perf_callchain_user() / perf_callchain_kernel() That avoid all the user_mode(), current->mm checks and so...
- Invert some parameters in perf_callchain_*() helpers: entry to the left, regs to the right, following the traditional (src, dst).
I'm not sure what stack_trace_flush() and flushw_user() do in sparc, I've just moved them inside the callchain_user handlers, I'm not sure this is the right thing.
--- arch/arm/kernel/perf_event.c | 43 +++--------------------------- arch/powerpc/kernel/perf_callchain.c | 49 +++++++++------------------------ arch/sh/kernel/perf_callchain.c | 37 +------------------------ arch/sparc/kernel/perf_event.c | 46 ++++++++++--------------------- arch/x86/kernel/cpu/perf_event.c | 45 +++++------------------------- include/linux/perf_event.h | 10 ++++++- kernel/perf_event.c | 40 ++++++++++++++++++++++++++- 7 files changed, 90 insertions(+), 180 deletions(-)
arch/arm/kernel/perf_event.c +4 -39
@@ -3044,17 +3044,13 @@ user_backtrace(struct frame_tail *tail,return buftail.fp -1 ; } -static void -perf_callchain_user (struct pt_regs *regs, -struct perf_callchain_entry *entry) +void +perf_callchain_user (struct perf_callchain_entry *entry,struct pt_regs *regs) {struct frame_tail *tail;perf_callchain_store (entry,PERF_CONTEXT_USER ); -if (!user_mode (regs)) - regs =task_pt_regs (current); - tail = (struct frame_tail *)regs->ARM_fp -1 ;while (tail && !((unsigned long )tail &0x3 ))
@@ -3075,9 +3071,8 @@ callchain_trace(struct stackframe *fr,return 0 ; } -static void -perf_callchain_kernel (struct pt_regs *regs, -struct perf_callchain_entry *entry) +void +perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs) {struct stackframe fr;
@@ -3088,33 +3083,3 @@ perf_callchain_kernel(struct pt_regs *regs, fr.pc = regs->ARM_pc;walk_stackframe (&fr, callchain_trace, entry); } - -static void -perf_do_callchain (struct pt_regs *regs, -struct perf_callchain_entry *entry) -{ -int is_user; - -if (!regs) -return ; - - is_user =user_mode (regs); - -if (!is_user) -perf_callchain_kernel (regs, entry); - -if (current->mm) -perf_callchain_user (regs, entry); -} - -static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry); - -struct perf_callchain_entry * -perf_callchain (struct pt_regs *regs) -{ -struct perf_callchain_entry *entry = &__get_cpu_var (pmc_irq_entry); - - entry->nr =0 ; -perf_do_callchain (regs, entry); -return entry; -}
arch/powerpc/kernel/perf_callchain.c +14 -35
78 lines not on this page · raw message
arch/sh/kernel/perf_callchain.c +2 -35
@@ -44,44 +44,11 @@ static const struct stacktrace_ops callchain_ops = { .address = callchain_address, }; -static void -perf_callchain_kernel (struct pt_regs *regs,struct perf_callchain_entry *entry) +void +perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs) {perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_store (entry, regs->pc);unwind_stack (NULL , regs,NULL , &callchain_ops, entry); } - -static void -perf_do_callchain (struct pt_regs *regs,struct perf_callchain_entry *entry) -{ -int is_user; - -if (!regs) -return ; - - is_user =user_mode (regs); - -/* -* Only the kernel side is implemented for now. -*/ -if (!is_user) -perf_callchain_kernel (regs, entry); -} - -/* -* No need for separate IRQ and NMI entries. -*/ -static DEFINE_PER_CPU(struct perf_callchain_entry, callchain); - -struct perf_callchain_entry *perf_callchain (struct pt_regs *regs) -{ -struct perf_callchain_entry *entry = &__get_cpu_var (callchain); - - entry->nr =0 ; - -perf_do_callchain (regs, entry); - -return entry; -}
arch/sparc/kernel/perf_event.c +15 -31
75 lines not on this page · raw message
arch/x86/kernel/cpu/perf_event.c +8 -37
81 lines not on this page · raw message
include/linux/perf_event.h +9 -1
@@ -976,7 +976,15 @@ extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacksextern void perf_event_comm (struct task_struct *tsk);extern void perf_event_fork (struct task_struct *tsk); -extern struct perf_callchain_entry *perf_callchain (struct pt_regs *regs); +/* Callchains */ +DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); + +extern void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs); +extern void perf_callchain_kernel (struct perf_callchain_entry *entry, +struct pt_regs *regs); +extern struct perf_callchain_entry *perf_callchain_buffer (void ); +static inline void perf_callchain_store (struct perf_callchain_entry *entry,u64 ip)
kernel/perf_event.c +38 -2
@@ -2937,13 +2937,49 @@ void perf_event_do_pending(void)__perf_pending_run (); } +DEFINE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); +/* * Callchain support -- arch specific */ -__weak struct perf_callchain_entry *perf_callchain (struct pt_regs *regs) +__weak struct perf_callchain_entry *perf_callchain_buffer (void ) { -return NULL ; +return &__get_cpu_var (perf_callchain_entry); +} + +__weak void perf_callchain_kernel (struct perf_callchain_entry *entry, +struct pt_regs *regs) +{ +} + +__weak void perf_callchain_user (struct perf_callchain_entry *entry, +struct pt_regs *regs) +{ +} + +static struct perf_callchain_entry *perf_callchain (struct pt_regs *regs) +{ +struct perf_callchain_entry *entry; + + entry =perf_callchain_buffer (); +if (!entry) +return NULL ; + + entry->nr =0 ; + +if (!user_mode (regs)) { +perf_callchain_kernel (entry, regs); +if (current->mm) + regs =task_pt_regs (current); +else + regs =NULL ; + } + +if (regs) +perf_callchain_user (entry, regs); + +return entry; }
Signature · 1 line
1.6.2.3
Peter Zijlstra
maintainerRe: [RFC PATCH 3/6] perf: Generalize some arch callchain code
Quoted 3 lines · Frederic Weisbecker
- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default
perf_callchain_buffer()implementation that x86 overrides.
sparc and power also have NMI like contexts.
Re: [RFC PATCH 3/6] perf: Generalize some arch callchain code
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default
perf_callchain_buffer()implementation that x86 overrides.sparc and power also have NMI like contexts.
Ah and the comments suggest it's because pmu interrupts can't nest or so. Anyway, that's notwithstanding the race that 5th patch fixes.
Peter Zijlstra
maintainerRe: [RFC PATCH 3/6] perf: Generalize some arch callchain code
Quoted 11 lines · Frederic Weisbecker
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default
perf_callchain_buffer()implementation that x86 overrides.sparc and power also have NMI like contexts.
Ah and the comments suggest it's because pmu interrupts can't nest or so. Anyway, that's notwithstanding the race that 5th patch fixes.
Right, but they could interrupt a software event or the like.
But yeah, the whole callchain thing can nest issue is real, thanks for fixing that.
Re: [RFC PATCH 3/6] perf: Generalize some arch callchain code
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Frederic Weisbecker
Quoted 5 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default
perf_callchain_buffer()implementation that x86 overrides.
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
sparc and power also have NMI like contexts.
Ah and the comments suggest it's because pmu interrupts can't nest or so. Anyway, that's notwithstanding the race that 5th patch fixes.
Right, but they could interrupt a software event or the like.
Yeah indeed.
Re: [RFC PATCH 3/6] perf: Generalize some arch callchain code
In reply to Peter ZijlstraQuoted 6 lines · Peter Zijlstra
Quoted 3 lines · Frederic Weisbecker
- Most archs use one callchain buffer per cpu, except x86 that needs to deal with NMIs. Provide a default
perf_callchain_buffer()implementation that x86 overrides.sparc and power also have NMI like contexts.
Right, but they didn't split that into different nested buffers. The 5th patch does anyway.
RFC PATCH 4/6 perf: Factorize callchain context handling
In reply to Frederic WeisbeckerStore the kernel and user contexts from the generic layer instead of archs, this gathers some repetitive code.
--- arch/arm/kernel/perf_event.c | 2 -- arch/powerpc/kernel/perf_callchain.c | 3 --- arch/sh/kernel/perf_callchain.c | 1 - arch/sparc/kernel/perf_event.c | 3 --- arch/x86/kernel/cpu/perf_event.c | 2 -- kernel/perf_event.c | 5 ++++- 6 files changed, 4 insertions(+), 12 deletions(-)
arch/arm/kernel/perf_event.c +0 -2
@@ -3049,7 +3049,6 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) {struct frame_tail *tail; -perf_callchain_store (entry,PERF_CONTEXT_USER ); tail = (struct frame_tail *)regs->ARM_fp -1 ;
@@ -3076,7 +3075,6 @@ perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) {struct stackframe fr; -perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); fr.fp = regs->ARM_fp; fr.sp = regs->ARM_sp; fr.lr = regs->ARM_lr;
arch/powerpc/kernel/perf_callchain.c +0 -3
@@ -57,7 +57,6 @@ perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) lr = regs->link; sp = regs->gpr[1 ]; -perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_store (entry, regs->nip);if (!validate_sp (sp, current,STACK_FRAME_OVERHEAD ))
@@ -234,7 +233,6 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, next_ip = regs->nip; lr = regs->link; sp = regs->gpr[1 ]; -perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_store (entry, next_ip);for (;;) {
@@ -435,7 +433,6 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, next_ip = regs->nip; lr = regs->link; sp = regs->gpr[1 ]; -perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_store (entry, next_ip);while (entry->nr <PERF_MAX_STACK_DEPTH ) {
arch/sh/kernel/perf_callchain.c +0 -1
@@ -47,7 +47,6 @@ static const struct stacktrace_ops callchain_ops = {void perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs) { -perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_store (entry, regs->pc);unwind_stack (NULL , regs,NULL , &callchain_ops, entry);
arch/sparc/kernel/perf_event.c +0 -3
@@ -1292,7 +1292,6 @@ void perf_callchain_kernel(struct perf_callchain_entry *entry,stack_trace_flush (); -perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_store (entry, regs->tpc); ksp = regs->u_regs[UREG_I6 ];
@@ -1336,7 +1335,6 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, {unsigned long ufp; -perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_store (entry, regs->tpc); ufp = regs->u_regs[UREG_I6 ] +STACK_BIAS ;
@@ -1359,7 +1357,6 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, {unsigned long ufp; -perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_store (entry, regs->tpc); ufp = regs->u_regs[UREG_I6 ] &0xffffffffUL ;
arch/x86/kernel/cpu/perf_event.c +0 -2
@@ -1608,7 +1608,6 @@ static const struct stacktrace_ops backtrace_ops = {void perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs) { -perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_store (entry, regs->ip);dump_trace (NULL , regs,NULL , regs->bp, &backtrace_ops, entry);
@@ -1660,7 +1659,6 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) fp = (void __user *)regs->bp; -perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_store (entry, regs->ip);if (perf_callchain_user32 (regs, entry))
kernel/perf_event.c +4 -1
@@ -2969,6 +2969,7 @@ static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) entry->nr =0 ;if (!user_mode (regs)) { +perf_callchain_store (entry,PERF_CONTEXT_KERNEL );perf_callchain_kernel (entry, regs);if (current->mm) regs =task_pt_regs (current);
@@ -2976,8 +2977,10 @@ static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) regs =NULL ; } -if (regs) +if (regs) { +perf_callchain_store (entry,PERF_CONTEXT_USER );perf_callchain_user (entry, regs); + }return entry; }
Signature · 1 line
1.6.2.3
RFC PATCH 5/6 perf: Fix race in callchains
In reply to Frederic WeisbeckerNow that software events don't have interrupt disabled anymore in the event path, callchains can nest on any context. So seperating nmi and others contexts in two buffers has become racy.
Fix this by providing one buffer per nesting level. Given the size of the callchain entries (2040 bytes * 4), we now need to allocate them dynamically.
(The guest checks in x86 should probably be moved elsewhere).
--- arch/x86/kernel/cpu/perf_event.c | 22 ++-- include/linux/perf_event.h | 1 - kernel/perf_event.c | 265 ++++++++++++++++++++++++++++---------- 3 files changed, 205 insertions(+), 83 deletions(-)
arch/x86/kernel/cpu/perf_event.c +9 -13
@@ -1608,6 +1608,11 @@ static const struct stacktrace_ops backtrace_ops = {void perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs) { +if (perf_guest_cbs && perf_guest_cbs->is_in_guest ()) { +/* TODO: We don't support guest os callchain now */ +return NULL ; + } +perf_callchain_store (entry, regs->ip);dump_trace (NULL , regs,NULL , regs->bp, &backtrace_ops, entry);
@@ -1656,6 +1661,10 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)struct stack_frame frame;const void __user *fp; +if (perf_guest_cbs && perf_guest_cbs->is_in_guest ()) { +/* TODO: We don't support guest os callchain now */ +return NULL ; + } fp = (void __user *)regs->bp;
@@ -1681,19 +1690,6 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) } } -struct perf_callchain_entry *perf_callchain_buffer (void ) -{ -if (perf_guest_cbs && perf_guest_cbs->is_in_guest ()) { -/* TODO: We don't support guest os callchain now */ -return NULL ; - } - -if (in_nmi ()) -return &__get_cpu_var (perf_callchain_entry_nmi); - -return &__get_cpu_var (perf_callchain_entry); -} -unsigned long perf_instruction_pointer (struct pt_regs *regs) {unsigned long ip;
include/linux/perf_event.h +0 -1
@@ -983,7 +983,6 @@ extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);extern void perf_callchain_kernel (struct perf_callchain_entry *entry,struct pt_regs *regs); -extern struct perf_callchain_entry *perf_callchain_buffer (void ); static inline void
kernel/perf_event.c +196 -69
315 lines not on this page · raw message
Signature · 1 line
1.6.2.3
Re: [RFC PATCH 5/6] perf: Fix race in callchains
Quoted 20 lines · Frederic Weisbecker
kernel/perf_event.c +20
-0+static struct perf_callchain_entry *get_callchain_entry (int *rctx) +{ +struct perf_callchain_entry_cpus *cpu_entries; + + *rctx =get_recursion_context (&__get_cpu_var (callchain_recursion)); +if (*rctx ==-1 ) +return NULL ; + + cpu_entries =rcu_dereference (callchain_entries[*rctx]); +if (!cpu_entries) +return NULL ; + +return this_cpu_ptr (cpu_entries->entries); +} + +static void +put_callchain_entry (int rctx) +{ +put_recursion_context (&__get_cpu_var (callchain_recursion), rctx); +}
kernel/perf_event.c static u64 perf_event_read(struct perf_event *event) · line 1919
Quoted 10 lines
kernel/perf_event.c +10
-0+ +static struct perf_callchain_entry *perf_callchain (struct pt_regs *regs) +{ +int rctx; +struct perf_callchain_entry *entry; + + + entry =get_callchain_entry (&rctx); +if (!entry) + goto exit_put;
I realize this should only call put_callchain_entry() if rctx == -1, but you got the idea...
Quoted 22 lines
kernel/perf_event.c +22
-0+ + entry->nr =0 ; + +if (!user_mode (regs)) { +perf_callchain_store (entry,PERF_CONTEXT_KERNEL ); +perf_callchain_kernel (entry, regs); +if (current->mm) + regs =task_pt_regs (current); +else + regs =NULL ; + } + +if (regs) { +perf_callchain_store (entry,PERF_CONTEXT_USER ); +perf_callchain_user (entry, regs); + } + +exit_put: +put_callchain_entry (rctx); + +return entry; +}
Peter Zijlstra
maintainerRe: [RFC PATCH 5/6] perf: Fix race in callchains
In reply to Frederic WeisbeckerQuoted 7 lines · Frederic Weisbecker
Now that software events don't have interrupt disabled anymore in the event path, callchains can nest on any context. So seperating nmi and others contexts in two buffers has become racy.
Fix this by providing one buffer per nesting level. Given the size of the callchain entries (2040 bytes * 4), we now need to allocate them dynamically.
OK so I guess you want to allocate them because 8k per cpu is too much to always have about?
Quoted 52 lines
kernel/perf_event.c +51
-0+static int get_callchain_buffers (void ) +{ +int i; +int err =0 ; +struct perf_callchain_entry_cpus *buf; + +mutex_lock (&callchain_mutex); + +if (WARN_ON_ONCE (++nr_callchain_events <1 )) { + err = -EINVAL ; + goto exit; + } + +if (nr_callchain_events >1 ) + goto exit; + +for (i =0 ; i <4 ; i++) { + buf =kzalloc (sizeof (*buf),GFP_KERNEL ); +/* free_event() will clean the rest */ +if (!buf) { + err = -ENOMEM ; + goto exit; + } + buf->entries = alloc_percpu(struct perf_callchain_entry); +if (!buf->entries) { +kfree (buf); + err = -ENOMEM ; + goto exit; + } +rcu_assign_pointer (callchain_entries[i], buf); + } + +exit: +mutex_unlock (&callchain_mutex); + +return err; +} +static void put_callchain_buffers (void ) +{ +int i; +struct perf_callchain_entry_cpus *entry; + +mutex_lock (&callchain_mutex); + +if (WARN_ON_ONCE (--nr_callchain_events <0 )) + goto exit; + +if (nr_callchain_events >0 ) + goto exit; + +for (i =0 ; i <4 ; i++) {
kernel/perf_event.c static u64 perf_event_read(struct perf_event *event) · line 1860
Quoted 10 lines
kernel/perf_event.c +10
-0+ entry = callchain_entries[i]; +if (entry) { + callchain_entries[i] =NULL ; +call_rcu (&entry->rcu_head, release_callchain_buffers); + } + } + +exit: +mutex_unlock (&callchain_mutex); +}
If you make nr_callchain_events an atomic_t, then you can do the refcounting outside the mutex. See the existing user of atomic_dec_and_mutex_lock().
I would also split it in get/put and alloc/free functions for clarity.
I'm not at all sure why you're using RCU though.
kernel/perf_event.c static void free_event(struct perf_event *event) · line 2079
Quoted 9 lines
kernel/perf_event.c +2
-0@@ -1895,6 +2072,8 @@ static void free_event(struct perf_event *event)atomic_dec (&nr_comm_events);if (event->attr.task)atomic_dec (&nr_task_events); +if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) +put_callchain_buffers (); }if (event->buffer) {
If this was the last even, there's no callchain user left, so nobody can be here:
Quoted 10 lines
kernel/perf_event.c +3
-0@@ -3480,14 +3610,20 @@ static void perf_event_output(struct perf_event *event, int nmi,struct perf_output_handle handle;struct perf_event_header header; +/* protect the callchain buffers */ +rcu_read_lock (); +perf_prepare_sample (&header, data, event, regs);if (perf_output_begin (&handle, event, header.size, nmi,1 ))
kernel/perf_event.c static void perf_event_output(struct perf_event *event, int nmi, · line 3627
Quoted 10 lines
kernel/perf_event.c +4
-1-return ; + goto exit;perf_output_sample (&handle, &header, data, event);perf_output_end (&handle); + +exit: +rcu_read_unlock (); }
Rendering that RCU stuff superfluous.
Re: [RFC PATCH 5/6] perf: Fix race in callchains
Quoted 11 lines · Peter Zijlstra
Quoted 7 lines · Frederic Weisbecker
Now that software events don't have interrupt disabled anymore in the event path, callchains can nest on any context. So seperating nmi and others contexts in two buffers has become racy.
Fix this by providing one buffer per nesting level. Given the size of the callchain entries (2040 bytes * 4), we now need to allocate them dynamically.
OK so I guess you want to allocate them because 8k per cpu is too much to always have about?
Right. I know that really adds complexity and I hesitated much before doing so. But I think that's quite necessary.
Quoted 67 lines
Quoted 52 lines
kernel/perf_event.c +51
-0+static int get_callchain_buffers (void ) +{ +int i; +int err =0 ; +struct perf_callchain_entry_cpus *buf; + +mutex_lock (&callchain_mutex); + +if (WARN_ON_ONCE (++nr_callchain_events <1 )) { + err = -EINVAL ; + goto exit; + } + +if (nr_callchain_events >1 ) + goto exit; + +for (i =0 ; i <4 ; i++) { + buf =kzalloc (sizeof (*buf),GFP_KERNEL ); +/* free_event() will clean the rest */ +if (!buf) { + err = -ENOMEM ; + goto exit; + } + buf->entries = alloc_percpu(struct perf_callchain_entry); +if (!buf->entries) { +kfree (buf); + err = -ENOMEM ; + goto exit; + } +rcu_assign_pointer (callchain_entries[i], buf); + } + +exit: +mutex_unlock (&callchain_mutex); + +return err; +} +static void put_callchain_buffers (void ) +{ +int i; +struct perf_callchain_entry_cpus *entry; + +mutex_lock (&callchain_mutex); + +if (WARN_ON_ONCE (--nr_callchain_events <0 )) + goto exit; + +if (nr_callchain_events >0 ) + goto exit; + +for (i =0 ; i <4 ; i++) {kernel/perf_event.c static u64 perf_event_read(struct perf_event *event) · line 1860
Quoted 10 lines
kernel/perf_event.c +10
-0+ entry = callchain_entries[i]; +if (entry) { + callchain_entries[i] =NULL ; +call_rcu (&entry->rcu_head, release_callchain_buffers); + } + } + +exit: +mutex_unlock (&callchain_mutex); +}If you make
nr_callchain_eventsanatomic_t, then you can do the refcounting outside the mutex. See the existing user ofatomic_dec_and_mutex_lock().
Quoted 1 line
I would also split it in get/put and alloc/free functions for clarity.
Ok I will.
Quoted 30 lines
I'm not at all sure why you're using RCU though.
kernel/perf_event.c static void free_event(struct perf_event *event) · line 2079
Quoted 9 lines
kernel/perf_event.c +2
-0@@ -1895,6 +2072,8 @@ static void free_event(struct perf_event *event)atomic_dec (&nr_comm_events);if (event->attr.task)atomic_dec (&nr_task_events); +if (event->attr.sample_type &PERF_SAMPLE_CALLCHAIN ) +put_callchain_buffers (); }if (event->buffer) {If this was the last even, there's no callchain user left, so nobody can be here:
Quoted 15 lines
kernel/perf_event.c +4
-1@@ -3480,14 +3610,20 @@ static void perf_event_output(struct perf_event *event, int nmi,struct perf_output_handle handle;struct perf_event_header header; +/* protect the callchain buffers */ +rcu_read_lock (); +perf_prepare_sample (&header, data, event, regs);if (perf_output_begin (&handle, event, header.size, nmi,1 )) -return ; + goto exit;perf_output_sample (&handle, &header, data, event);
Quoted 7 lines
kernel/perf_event.c static void perf_event_output(struct perf_event *event, int nmi, · line 3627
Quoted 5 lines
kernel/perf_event.c +3
-0perf_output_end (&handle); + +exit: +rcu_read_unlock (); }Rendering that RCU stuff superfluous.
May be I'm omitting something that would make it non-rcu-safe.
But consider a perf event running on CPU 1. And you close the fd on CPU 0. CPU 1 has started to use a callchain buffer but receives an IPI to retire the event from the cpu. But still it has yet to finish his callchain processing.
If right after that CPU 0 releases the callchain buffers, CPU 1 may crash in the middle.
So you need to wait for the grace period to end.
RFC PATCH 6/6 perf: Fix double put_ctx
In reply to Frederic WeisbeckerIf we call free_event on fail case of event creation, it already put the context. The falldown goto, though, also does a put_ctx, which might dereference a freed context.
--- kernel/perf_event.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
kernel/perf_event.c +2 -0
@@ -5364,6 +5364,8 @@ SYSCALL_DEFINE5(perf_event_open, err_free_put_context:free_event (event); +fput_light (group_file, fput_needed); + goto err_fd; err_put_context:fput_light (group_file, fput_needed);put_ctx (ctx);
Signature · 1 line
1.6.2.3