Notify page fault call chain
cover Notify page fault call chain
Hi Andrew,
Currently in the do_page_fault() code path, we call notify_die(DIE_PAGE_FAULT, ...) to notify the page fault. The only interested components for this page fault notifications are Kprobes and/or kdb. Since notify_die() is highly overloaded, this page fault notification is currently being sent to other components registered with register_die_notification() which uses the same die_chain to loop for all the registered components.
In order to optimize the do_page_fault() code path, this critical page fault notification is now moved to different call chain and the test results showed great improvements.
Patches for i386, x86_64, ia64, powerpc and sparc64 follows this mail.
Please apply.
Thanks,
Anil Keshavamurthy
--
PATCH 1/6 Notify page fault call chain for i386
--- arch/i386/kernel/traps.c | 12 ++++++++++++ arch/i386/mm/fault.c | 4 ++-- include/asm-i386/kdebug.h | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-)
arch/i386/kernel/traps.c +12 -0
@@ -93,6 +93,7 @@ asmlinkage void machine_check(void);static int kstack_depth_to_print =24 ;ATOMIC_NOTIFIER_HEAD (i386die_chain); +ATOMIC_NOTIFIER_HEAD (notify_page_fault_chain);int register_die_notifier (struct notifier_block *nb) {
@@ -107,6 +108,17 @@ int unregister_die_notifier(struct notif }EXPORT_SYMBOL (unregister_die_notifier); +int register_page_fault_notifier (struct notifier_block *nb) +{ +vmalloc_sync_all (); +return atomic_notifier_chain_register (¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_unregister (¬ify_page_fault_chain, nb); +} +static inline int valid_stack_ptr (struct thread_info *tinfo,void *p) { return p > (void *)tinfo &&
arch/i386/mm/fault.c +2 -2
@@ -321,7 +321,7 @@ fastcall void __kprobes do_page_fault(stif (unlikely (address >=TASK_SIZE )) {if (!(error_code &0x0000000d ) &&vmalloc_fault (address) >=0 )return ; -if (notify_die (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 , +if (notify_page_fault (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 ,SIGSEGV ) ==NOTIFY_STOP )return ;/*
@@ -331,7 +331,7 @@ fastcall void __kprobes do_page_fault(st goto bad_area_nosemaphore; } -if (notify_die (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 , +if (notify_page_fault (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 ,SIGSEGV ) ==NOTIFY_STOP )return ;
include/asm-i386/kdebug.h +15 -0
@@ -19,7 +19,10 @@ struct die_args {extern int register_die_notifier (struct notifier_block *);extern int unregister_die_notifier (struct notifier_block *); +extern int register_page_fault_notifier (struct notifier_block *); +extern int unregister_page_fault_notifier (struct notifier_block *);extern struct atomic_notifier_head i386die_chain; +extern struct atomic_notifier_head notify_page_fault_chain;/* Grossly misnamed. */
@@ -53,4 +56,16 @@ static inline int notify_die(enum die_va return atomic_notifier_call_chain(&i386die_chain, val, &args); } +static inline int notify_page_fault (enum die_val val,const char *str, +struct pt_regs *regs,long err,int trap,int sig) +{ +struct die_args args = { + .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig + }; +return atomic_notifier_call_chain (¬ify_page_fault_chain, val, &args); +}#endif
--
PATCH 2/6 Notify page fault call chain for x86_64
In reply to Anil S Keshavamurthy--- arch/x86_64/kernel/traps.c | 12 ++++++++++++ arch/x86_64/mm/fault.c | 4 ++-- include/asm-x86_64/kdebug.h | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-)
arch/x86_64/kernel/traps.c +12 -0
@@ -71,6 +71,7 @@ asmlinkage void machine_check(void); asmlinkage void spurious_interrupt_bug(void);ATOMIC_NOTIFIER_HEAD (die_chain); +ATOMIC_NOTIFIER_HEAD (notify_page_fault_chain);int register_die_notifier (struct notifier_block *nb) {
@@ -85,6 +86,17 @@ int unregister_die_notifier(struct notif }EXPORT_SYMBOL (unregister_die_notifier); +int register_page_fault_notifier (struct notifier_block *nb) +{ +vmalloc_sync_all (); +return atomic_notifier_chain_register (¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_unregister (¬ify_page_fault_chain, nb); +} +static inline void conditional_sti (struct pt_regs *regs) {if (regs->eflags &X86_EFLAGS_IF )
arch/x86_64/mm/fault.c +2 -2
@@ -348,7 +348,7 @@ asmlinkage void __kprobes do_page_fault(if (vmalloc_fault (address) >=0 )return ; } -if (notify_die (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 , +if (notify_page_fault (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 ,SIGSEGV ) ==NOTIFY_STOP )return ;/*
@@ -358,7 +358,7 @@ asmlinkage void __kprobes do_page_fault( goto bad_area_nosemaphore; } -if (notify_die (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 , +if (notify_page_fault (DIE_PAGE_FAULT ,"page fault" , regs, error_code,14 ,SIGSEGV ) ==NOTIFY_STOP )return ;
include/asm-x86_64/kdebug.h +16 -0
@@ -15,7 +15,10 @@ struct die_args {extern int register_die_notifier (struct notifier_block *);extern int unregister_die_notifier (struct notifier_block *); +extern int register_page_fault_notifier (struct notifier_block *); +extern int unregister_page_fault_notifier (struct notifier_block *);extern struct atomic_notifier_head die_chain; +extern struct atomic_notifier_head notify_page_fault_chain;/* Grossly misnamed. */ enum die_val {
@@ -47,6 +50,19 @@ static inline int notify_die(enum die_vareturn atomic_notifier_call_chain (&die_chain, val, &args); } +static inline int notify_page_fault (enum die_val val,const char *str, +struct pt_regs *regs,long err,int trap,int sig) +{ +struct die_args args = { + .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig + }; +return atomic_notifier_call_chain (¬ify_page_fault_chain, val, &args); +} +extern int printk_address (unsigned long address);extern void die (const char *,struct pt_regs *,long );extern void __die (const char *,struct pt_regs *,long );
--
Re: [patch 2/6] Notify page fault call chain for x86_64
On Wednesday 19 April 2006 21:01, Anil S Keshavamurthy wrote:
You seem to be missing a description/rationale here.
-Andi
Quoted 5 lines
---
arch/x86_64/kernel/traps.c | 12 ++++++++++++ arch/x86_64/mm/fault.c | 4 ++-- include/asm-x86_64/kdebug.h | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-)
Re: [patch 2/6] Notify page fault call chain for x86_64
Quoted 3 lines · Andi Kleen
On Wednesday 19 April 2006 21:01, Anil S Keshavamurthy wrote:
You seem to be missing a description/rationale here.
Sorry about that, I had the description in [patch 0/6] and looks like I missed you cc'ing in that patch. Any way I am copying it again here.
"Currently in the do_page_fault() code path, we call
notify_die(DIE_PAGE_FAULT, ...) to notify the page fault. The only interested components for this page fault notifications are Kprobes and/or kdb. Since notify_die() is highly overloaded, this page fault notification is currently being sent to other components
registered with register_die_notification() which uses the same
die_chain to loop for all the registered components.
In order to optimize the do_page_fault() code path, this critical page fault notification is now moved to different call chain and the test results showed great improvements.
Patches for i386, x86_64, ia64, powerpc and sparc64 follows this mail."
This patch introduces the exclusive {register/unregister}_page_fault_notifier() for use by Kprobes and kdb.
-Anil
PATCH 3/6 Notify page fault call chain for ia64
In reply to Anil S Keshavamurthy--- arch/ia64/kernel/traps.c | 11 +++++++++++ arch/ia64/mm/fault.c | 2 +- include/asm-ia64/kdebug.h | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-)
arch/ia64/kernel/traps.c +11 -0
@@ -31,6 +31,7 @@ fpswa_interface_t *fpswa_interface;EXPORT_SYMBOL (fpswa_interface);ATOMIC_NOTIFIER_HEAD (ia64die_chain); +ATOMIC_NOTIFIER_HEAD (notify_page_fault_chain);int register_die_notifier (struct notifier_block *nb)
@@ -46,6 +47,16 @@ unregister_die_notifier(struct notifier_ }EXPORT_SYMBOL_GPL (unregister_die_notifier); +int register_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_register (¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_unregister (¬ify_page_fault_chain, nb); +} +void __init trap_init (void ) {
arch/ia64/mm/fault.c +1 -1
@@ -84,7 +84,7 @@ ia64_do_page_fault (unsigned long addres/* * This is to handle the kprobes on user space access instructions */ -if (notify_die (DIE_PAGE_FAULT ,"page fault" , regs, code,TRAP_BRKPT , +if (notify_page_fault (DIE_PAGE_FAULT ,"page fault" , regs, code,TRAP_BRKPT ,SIGSEGV ) ==NOTIFY_STOP )return ;
include/asm-ia64/kdebug.h +16 -0
@@ -40,7 +40,10 @@ struct die_args {extern int register_die_notifier (struct notifier_block *);extern int unregister_die_notifier (struct notifier_block *); +extern int register_page_fault_notifier (struct notifier_block *); +extern int unregister_page_fault_notifier (struct notifier_block *);extern struct atomic_notifier_head ia64die_chain; +extern struct atomic_notifier_head notify_page_fault_chain;enum die_val {DIE_BREAK =1 ,
@@ -86,4 +89,17 @@ static inline int notify_die(enum die_va return atomic_notifier_call_chain(&ia64die_chain, val, &args); } +static inline int notify_page_fault (enum die_val val,char *str,struct pt_regs *regs, +long err,int trap,int sig) +{ +struct die_args args = { + .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig + }; + +return atomic_notifier_call_chain (¬ify_page_fault_chain, val, &args); +}#endif
--
PATCH 4/6 Notify page fault call chain for powerpc
In reply to Anil S Keshavamurthy--- arch/powerpc/kernel/traps.c | 11 +++++++++++ arch/powerpc/mm/fault.c | 2 +- include/asm-powerpc/kdebug.h | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-)
arch/powerpc/kernel/traps.c +11 -0
@@ -75,6 +75,7 @@ EXPORT_SYMBOL(__debugger_fault_handler);#endif ATOMIC_NOTIFIER_HEAD (powerpc_die_chain); +ATOMIC_NOTIFIER_HEAD (notify_page_fault_chain);int register_die_notifier (struct notifier_block *nb) {
@@ -88,6 +89,16 @@ int unregister_die_notifier(struct notif }EXPORT_SYMBOL (unregister_die_notifier); +int register_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_register (¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_unregister (¬ify_page_fault_chain, nb); +} +/* * Trap & Exception support */
arch/powerpc/mm/fault.c +1 -1
@@ -142,7 +142,7 @@ int __kprobes do_page_fault(struct pt_re is_write = error_code &ESR_DST ;#endif /* CONFIG_4xx || CONFIG_BOOKE */ -if (notify_die (DIE_PAGE_FAULT ,"page_fault" , regs, error_code, +if (notify_page_fault (DIE_PAGE_FAULT ,"page_fault" , regs, error_code,11 ,SIGSEGV ) ==NOTIFY_STOP )return 0 ;
include/asm-powerpc/kdebug.h +9 -0
@@ -18,7 +18,10 @@ struct die_args {extern int register_die_notifier (struct notifier_block *);extern int unregister_die_notifier (struct notifier_block *); +extern int register_page_fault_notifier (struct notifier_block *); +extern int unregister_page_fault_notifier (struct notifier_block *);extern struct atomic_notifier_head powerpc_die_chain; +extern struct atomic_notifier_head notify_page_fault_chain;/* Grossly misnamed. */ enum die_val {
@@ -36,5 +39,11 @@ static inline int notify_die(enum die_vareturn atomic_notifier_call_chain (&powerpc_die_chain, val, &args); } +static inline int notify_page_fault (enum die_val val,char *str,struct pt_regs *regs,long err,int trap,int sig) +{ +struct die_args args = { .regs=regs, .str=str, .err=err, .trapnr=trap,.signr=sig }; +return atomic_notifier_call_chain (¬ify_page_fault_chain, val, &args); +} +#endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_KDEBUG_H */
--
PATCH 5/6 Notify page fault call chain for sparc64
In reply to Anil S Keshavamurthy--- arch/sparc64/kernel/traps.c | 11 +++++++++++ arch/sparc64/mm/fault.c | 2 +- include/asm-sparc64/kdebug.h | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-)
arch/sparc64/kernel/traps.c +11 -0
@@ -44,6 +44,7 @@#endif ATOMIC_NOTIFIER_HEAD (sparc64die_chain); +ATOMIC_NOTIFIER_HEAD (notify_page_fault_chain);int register_die_notifier (struct notifier_block *nb) {
@@ -57,6 +58,16 @@ int unregister_die_notifier(struct notif }EXPORT_SYMBOL (unregister_die_notifier); +int register_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_register (¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier (struct notifier_block *nb) +{ +return atomic_notifier_chain_unregister (¬ify_page_fault_chain, nb); +} +/* When an irrecoverable trap occurs at tl > 0, the trap entry * code logs the trap state registers at every level in the trap * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
arch/sparc64/mm/fault.c +1 -1
@@ -263,7 +263,7 @@ asmlinkage void __kprobes do_sparc64_fau fault_code =get_thread_fault_code (); -if (notify_die (DIE_PAGE_FAULT ,"page_fault" , regs, +if (notify_page_fault (DIE_PAGE_FAULT ,"page_fault" , regs, fault_code,0 ,SIGSEGV ) ==NOTIFY_STOP )return ;
include/asm-sparc64/kdebug.h +16 -0
@@ -17,7 +17,10 @@ struct die_args {extern int register_die_notifier (struct notifier_block *);extern int unregister_die_notifier (struct notifier_block *); +extern int register_page_fault_notifier (struct notifier_block *); +extern int unregister_page_fault_notifier (struct notifier_block *);extern struct atomic_notifier_head sparc64die_chain; +extern struct atomic_notifier_head notify_page_fault_chain;extern void bad_trap (struct pt_regs *,long );
@@ -46,4 +49,17 @@ static inline int notify_die(enum die_va return atomic_notifier_call_chain(&sparc64die_chain, val, &args); } +static inline int notify_page_fault (enum die_val val,char *str,struct pt_regs *regs, +long err,int trap,int sig) +{ +struct die_args args = { .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig }; + +return atomic_notifier_call_chain (¬ify_page_fault_chain, val, &args); +} + +#endif
--
PATCH 6/6 Kprobes registers for notify page fault
In reply to Anil S KeshavamurthyKprobes now registers for the page fault notifications.
--- kernel/kprobes.c | 8 ++++++++ 1 file changed, 8 insertions(+)
kernel/kprobes.c +8 -0
@@ -544,6 +544,11 @@ static struct notifier_block kprobe_exce .priority = 0x7fffffff /* we need to notified first */ }; +static struct notifier_block kprobe_page_fault_nb = { + .notifier_call = kprobe_exceptions_notify, + .priority =0x7fffffff /* we need to notified first */ +}; +int __kprobes register_jprobe (struct jprobe *jp) {/* Todo: Verify probepoint is a function entry point */
@@ -654,6 +659,9 @@ static int __init init_kprobes(void)if (!err) err =register_die_notifier (&kprobe_exceptions_nb); +if (!err) + err =register_page_fault_notifier (&kprobe_page_fault_nb); +return err; }
--