Add support for TIF_NOTIFY_SIGNAL
Subsystems (25)
Jens Axboe
maintainerPATCH v4 1/1 Add support for TIF_NOTIFY_SIGNAL
Hi,
The goal is this patch series is to decouple TWA_SIGNAL based task_work from real signals and signal delivery. The motivation is speeding up TWA_SIGNAL based task_work, particularly for threaded setups where ->sighand is shared across threads. See the last patch for numbers.
v4 is nicely reduced, thanks to feedback from Oleg, dropping two of the core patches and resulting in something that is easier to adopt in other archs as well.
arch/alpha/kernel/signal.c | 1 - arch/arc/kernel/signal.c | 2 +- arch/arm/kernel/signal.c | 1 - arch/arm64/kernel/signal.c | 1 - arch/c6x/kernel/signal.c | 4 +-- arch/csky/kernel/signal.c | 1 - arch/h8300/kernel/signal.c | 4 +-- arch/hexagon/kernel/process.c | 1 - arch/ia64/kernel/process.c | 2 +- arch/m68k/kernel/signal.c | 2 +- arch/microblaze/kernel/signal.c | 2 +- arch/mips/kernel/signal.c | 1 - arch/nds32/kernel/signal.c | 4 +-- arch/nios2/kernel/signal.c | 2 +- arch/openrisc/kernel/signal.c | 1 - arch/parisc/kernel/signal.c | 4 +-- arch/powerpc/kernel/signal.c | 1 - arch/riscv/kernel/signal.c | 4 +-- arch/s390/kernel/signal.c | 1 - arch/sh/kernel/signal_32.c | 4 +-- arch/sparc/kernel/signal_32.c | 4 +-- arch/sparc/kernel/signal_64.c | 4 +-- arch/um/kernel/process.c | 2 +- arch/x86/include/asm/thread_info.h | 2 ++ arch/x86/kernel/signal.c | 5 +++- arch/xtensa/kernel/signal.c | 2 +- include/linux/entry-common.h | 6 ++++- include/linux/entry-kvm.h | 4 +-- include/linux/sched/signal.h | 20 ++++++++++++--- include/linux/tracehook.h | 31 ++++++++++++++++++++-- kernel/entry/common.c | 3 +-- kernel/entry/kvm.c | 7 ++--- kernel/events/uprobes.c | 2 +- kernel/signal.c | 8 +++--- kernel/task_work.c | 41 +++++++++++++++++++++--------- 35 files changed, 113 insertions(+), 71 deletions(-)
Also find the changes here:
https://git.kernel.dk/cgit/linux-block/log/?h=tif-task_work
Changes since v3:
- Drop not needed io_uring change
- Drop syscall restart split, handle TIF_NOTIFY_SIGNAL from the arch signal handling, using task_sigpending() to see if we need to care about real signals.
- Fix a few over-zelaous task_sigpending() changes
- Cleanup WARN_ON() in restore_saved_sigmask_unless()
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 tracehook: clear TIF_NOTIFY_RESUME in tracehook_notify_resume()
All the callers currently do this, clean it up and move the clearing into tracehook_notify_resume() instead.
--- arch/alpha/kernel/signal.c | 1 - arch/arc/kernel/signal.c | 2 +- arch/arm/kernel/signal.c | 1 - arch/arm64/kernel/signal.c | 1 - arch/c6x/kernel/signal.c | 4 +--- arch/csky/kernel/signal.c | 1 - arch/h8300/kernel/signal.c | 4 +--- arch/hexagon/kernel/process.c | 1 - arch/ia64/kernel/process.c | 2 +- arch/m68k/kernel/signal.c | 2 +- arch/microblaze/kernel/signal.c | 2 +- arch/mips/kernel/signal.c | 1 - arch/nds32/kernel/signal.c | 4 +--- arch/nios2/kernel/signal.c | 2 +- arch/openrisc/kernel/signal.c | 1 - arch/parisc/kernel/signal.c | 4 +--- arch/powerpc/kernel/signal.c | 1 - arch/riscv/kernel/signal.c | 4 +--- arch/s390/kernel/signal.c | 1 - arch/sh/kernel/signal_32.c | 4 +--- arch/sparc/kernel/signal_32.c | 4 +--- arch/sparc/kernel/signal_64.c | 4 +--- arch/um/kernel/process.c | 2 +- arch/xtensa/kernel/signal.c | 2 +- include/linux/tracehook.h | 4 ++-- kernel/entry/common.c | 1 - kernel/entry/kvm.c | 4 +--- 27 files changed, 18 insertions(+), 46 deletions(-)
arch/alpha/kernel/signal.c +0 -1
@@ -531,7 +531,6 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags,do_signal (regs, r0, r19); r0 =0 ; }else { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs); } }
arch/arc/kernel/signal.c +1 -1
@@ -394,6 +394,6 @@ void do_notify_resume(struct pt_regs *regs)* ASM glue gaurantees that this is only called when returning to * user mode */ -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); }
arch/arm/kernel/signal.c +0 -1
@@ -669,7 +669,6 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) }else if (thread_flags & _TIF_UPROBE) {uprobe_notify_resume (regs); }else { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/arm64/kernel/signal.c +0 -1
@@ -936,7 +936,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs,do_signal (regs);if (thread_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/c6x/kernel/signal.c +1 -3
@@ -316,8 +316,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags,if (thread_info_flags & (1 <<TIF_SIGPENDING ))do_signal (regs, syscall); -if (thread_info_flags & (1 <<TIF_NOTIFY_RESUME )) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & (1 <<TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); - } }
arch/csky/kernel/signal.c +0 -1
@@ -261,7 +261,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs,do_signal (regs);if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/h8300/kernel/signal.c +1 -3
@@ -282,8 +282,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags)if (thread_info_flags & _TIF_SIGPENDING)do_signal (regs); -if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - } }
arch/hexagon/kernel/process.c +0 -1
@@ -180,7 +180,6 @@ int do_work_pending(struct pt_regs *regs, u32 thread_info_flags) }if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);return 1 ; }
arch/ia64/kernel/process.c +1 -1
@@ -189,7 +189,7 @@ do_notify_resume_user(sigset_t *unused, struct sigscratch *scr, long in_syscall)ia64_do_signal (scr, in_syscall); } -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) { +if (test_thread_flag (TIF_NOTIFY_RESUME )) {local_irq_enable ();/* force interrupt enable */ tracehook_notify_resume (&scr->pt); }
arch/m68k/kernel/signal.c +1 -1
@@ -1134,6 +1134,6 @@ void do_notify_resume(struct pt_regs *regs)if (test_thread_flag (TIF_SIGPENDING ))do_signal (regs); -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); }
arch/microblaze/kernel/signal.c +1 -1
@@ -316,6 +316,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall)if (test_thread_flag (TIF_SIGPENDING ))do_signal (regs, in_syscall); -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); }
arch/mips/kernel/signal.c +0 -1
@@ -901,7 +901,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused,do_signal (regs);if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/nds32/kernel/signal.c +1 -3
@@ -379,8 +379,6 @@ do_notify_resume(struct pt_regs *regs, unsigned int thread_flags)if (thread_flags & _TIF_SIGPENDING)do_signal (regs); -if (thread_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - } }
arch/nios2/kernel/signal.c +1 -1
@@ -317,7 +317,7 @@ asmlinkage int do_notify_resume(struct pt_regs *regs)*/ return restart; } - }else if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) + }else if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs);return 0 ;
arch/openrisc/kernel/signal.c +0 -1
@@ -311,7 +311,6 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) } syscall =0 ; }else { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs); } }
arch/parisc/kernel/signal.c +1 -3
@@ -606,8 +606,6 @@ void do_notify_resume(struct pt_regs *regs, long in_syscall)if (test_thread_flag (TIF_SIGPENDING ))do_signal (regs, in_syscall); -if (test_thread_flag (TIF_NOTIFY_RESUME )) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); - } }
arch/powerpc/kernel/signal.c +0 -1
@@ -327,7 +327,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) }if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/riscv/kernel/signal.c +1 -3
@@ -313,8 +313,6 @@ asmlinkage __visible void do_notify_resume(struct pt_regs *regs,if (thread_info_flags & _TIF_SIGPENDING)do_signal (regs); -if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - } }
arch/s390/kernel/signal.c +0 -1
@@ -535,7 +535,6 @@ void do_signal(struct pt_regs *regs)void do_notify_resume (struct pt_regs *regs) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
arch/sh/kernel/signal_32.c +1 -3
@@ -502,8 +502,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned int save_r0,if (thread_info_flags & _TIF_SIGPENDING)do_signal (regs, save_r0); -if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - } }
arch/sparc/kernel/signal_32.c +1 -3
@@ -523,10 +523,8 @@ void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, {if (thread_info_flags & _TIF_SIGPENDING)do_signal (regs, orig_i0); -if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - } } asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr,
arch/sparc/kernel/signal_64.c +1 -3
@@ -551,10 +551,8 @@ void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned longuprobe_notify_resume (regs);if (thread_info_flags & _TIF_SIGPENDING)do_signal (regs, orig_i0); -if (thread_info_flags & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (thread_info_flags & _TIF_NOTIFY_RESUME)tracehook_notify_resume (regs); - }user_enter (); }
arch/um/kernel/process.c +1 -1
@@ -101,7 +101,7 @@ void interrupt_end(void)schedule ();if (test_thread_flag (TIF_SIGPENDING ))do_signal (regs); -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); }
arch/xtensa/kernel/signal.c +1 -1
@@ -501,6 +501,6 @@ void do_notify_resume(struct pt_regs *regs)if (test_thread_flag (TIF_SIGPENDING ))do_signal (regs); -if (test_and_clear_thread_flag (TIF_NOTIFY_RESUME )) +if (test_thread_flag (TIF_NOTIFY_RESUME ))tracehook_notify_resume (regs); }
include/linux/tracehook.h +2 -2
@@ -178,9 +178,9 @@ static inline void set_notify_resume(struct task_struct *task)*/ static inline void tracehook_notify_resume (struct pt_regs *regs) { +clear_thread_flag (TIF_NOTIFY_RESUME );/* -* The caller just cleared TIF_NOTIFY_RESUME. This barrier -* pairs with task_work_add()->set_notify_resume() after +* This barrier pairs with task_work_add()->set_notify_resume() after * hlist_add_head(task->task_works); */ smp_mb__after_atomic ();
kernel/entry/common.c +0 -1
@@ -161,7 +161,6 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,arch_do_signal (regs);if (ti_work & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME );tracehook_notify_resume (regs);rseq_handle_notify_resume (NULL , regs); }
kernel/entry/kvm.c +1 -3
@@ -16,10 +16,8 @@ static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work)if (ti_work & _TIF_NEED_RESCHED)schedule (); -if (ti_work & _TIF_NOTIFY_RESUME) { -clear_thread_flag (TIF_NOTIFY_RESUME ); +if (ti_work & _TIF_NOTIFY_RESUME)tracehook_notify_resume (NULL ); - } ret =arch_xfer_to_guest_mode_handle_work (vcpu, ti_work);if (ret)
Signature · 1 line
2.28.0
Re: [PATCH 1/4] tracehook: clear TIF_NOTIFY_RESUME in tracehook_notify_resume()
Quoted 4 lines · Jens Axboe
All the callers currently do this, clean it up and move the clearing into
tracehook_notify_resume()instead.Signed-off-by Jens Axboe <[email protected]>
Nice cleanup!
Jens Axboe
maintainerRe: [PATCH 1/4] tracehook: clear TIF_NOTIFY_RESUME in tracehook_notify_resume()
Quoted 9 lines · Thomas Gleixner
Quoted 4 lines · Jens Axboe
All the callers currently do this, clean it up and move the clearing into
tracehook_notify_resume()instead.Signed-off-by Jens Axboe <[email protected]>Nice cleanup!
Reviewed-by Thomas Gleixner <[email protected]>
Thanks! Added.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 2/4 kernel: add task_sigpending() helper
In reply to Jens AxboeThis is in preparation for maintaining signal_pending() as the decider of whether or not a schedule() loop should be broken, or continue sleeping. This is different than the core signal use cases, where we really want to know if an actual signal is pending or not. task_sigpending() returns non-zero if TIF_SIGPENDING is set.
Only core kernel use cases should care about the distinction between the two, make sure those use the task_sigpending() helper.
--- include/linux/sched/signal.h | 9 +++++++-- kernel/events/uprobes.c | 2 +- kernel/signal.c | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-)
include/linux/sched/signal.h +7 -2
@@ -353,11 +353,16 @@ static inline int restart_syscall(void)return -ERESTARTNOINTR ; } -static inline int signal_pending (struct task_struct *p) +static inline int task_sigpending (struct task_struct *p) {return unlikely (test_tsk_thread_flag (p,TIF_SIGPENDING )); } +static inline int signal_pending (struct task_struct *p) +{ +return task_sigpending (p); +} +static inline int __fatal_signal_pending (struct task_struct *p) {return unlikely (sigismember (&p->pending.signal,SIGKILL ));
@@ -365,7 +370,7 @@ static inline int __fatal_signal_pending(struct task_struct *p)static inline int fatal_signal_pending (struct task_struct *p) { -return signal_pending (p) &&__fatal_signal_pending (p); +return task_sigpending (p) &&__fatal_signal_pending (p); }static inline int signal_pending_state (long state,struct task_struct *p)
kernel/events/uprobes.c +1 -1
@@ -1973,7 +1973,7 @@ bool uprobe_deny_signal(void)WARN_ON_ONCE (utask->state !=UTASK_SSTEP ); -if (signal_pending (t)) { +if (task_sigpending (t)) {spin_lock_irq (&t->sighand->siglock);clear_tsk_thread_flag (t,TIF_SIGPENDING );spin_unlock_irq (&t->sighand->siglock);
kernel/signal.c +4 -4
@@ -983,7 +983,7 @@ static inline bool wants_signal(int sig, struct task_struct *p)if (task_is_stopped_or_traced (p))return false; -return task_curr (p) || !signal_pending (p); +return task_curr (p) || !task_sigpending (p); }static void complete_signal (int sig,struct task_struct *p,enum pid_type type)
@@ -2822,7 +2822,7 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)/* Remove the signals this thread can handle. */ sigandsets (&retarget, &retarget, &t->blocked); -if (!signal_pending (t)) +if (!task_sigpending (t))signal_wake_up (t,0 );if (sigisemptyset (&retarget))
@@ -2856,7 +2856,7 @@ void exit_signals(struct task_struct *tsk)cgroup_threadgroup_change_end (tsk); -if (!signal_pending (tsk)) +if (!task_sigpending (tsk)) goto out; unblocked = tsk->blocked;
@@ -2900,7 +2900,7 @@ long do_no_restart_syscall(struct restart_block *param)static void __set_task_blocked (struct task_struct *tsk,const sigset_t *newset) { -if (signal_pending (tsk) && !thread_group_empty (tsk)) { +if (task_sigpending (tsk) && !thread_group_empty (tsk)) {sigset_t newblocked;/* A set of now blocked but previously unblocked signals. */ sigandnsets (&newblocked, newset, ¤t->blocked);
Signature · 1 line
2.28.0
Re: [PATCH 2/4] kernel: add task_sigpending() helper
Quoted 10 lines · Jens Axboe
This is in preparation for maintaining
signal_pending()as the decider of whether or not aschedule()loop should be broken, or continue sleeping. This is different than the core signal use cases, where we really want to know if an actual signal is pending or not.task_sigpending()returns non-zero ifTIF_SIGPENDINGis set.Only core kernel use cases should care about the distinction between the two, make sure those use the
task_sigpending()helper.Signed-off-by Jens Axboe <[email protected]>
Jens Axboe
maintainerPATCH 3/4 kernel: add support for TIF_NOTIFY_SIGNAL
In reply to Jens AxboeThis adds TIF_NOTIFY_SIGNAL handling in the generic code, which if set, will return true if signal_pending() is used in a wait loop. That causes an exit of the loop so that notify_signal tracehooks can be run. If the wait loop is currently inside a system call, the system call is restarted once task_work has been processed.
x86 is using the generic entry code, add the necessary TIF_NOTIFY_SIGNAL definitions for it.
--- arch/x86/include/asm/thread_info.h | 2 ++ arch/x86/kernel/signal.c | 5 ++++- include/linux/entry-common.h | 6 +++++- include/linux/entry-kvm.h | 4 ++-- include/linux/sched/signal.h | 11 ++++++++++- include/linux/tracehook.h | 27 +++++++++++++++++++++++++++ kernel/entry/common.c | 2 +- kernel/entry/kvm.c | 3 +++ 8 files changed, 54 insertions(+), 6 deletions(-)
arch/x86/include/asm/thread_info.h +2 -0
@@ -93,6 +93,7 @@ struct thread_info {#define TIF_NOTSC 16/* TSC is not accessible in userland */ #define TIF_IA32 17/* IA32 compatibility process */ #define TIF_SLD 18/* Restore split lock detection on context switch */ +#define TIF_NOTIFY_SIGNAL 19/* signal notifications exist */ #define TIF_MEMDIE 20/* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21/* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22/* uses I/O bitmap */
@@ -123,6 +124,7 @@ struct thread_info {#define _TIF_NOTSC (1 << TIF_NOTSC)#define _TIF_IA32 (1 << TIF_IA32)#define _TIF_SLD (1 << TIF_SLD) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)#define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG)#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP)#define _TIF_FORCED_TF (1 << TIF_FORCED_TF)
arch/x86/kernel/signal.c +4 -1
@@ -808,7 +808,10 @@ void arch_do_signal(struct pt_regs *regs) {struct ksignal ksig; -if (get_signal (&ksig)) { +if (test_thread_flag (TIF_NOTIFY_SIGNAL )) +tracehook_notify_signal (); + +if (task_sigpending (current) &&get_signal (&ksig)) {/* Whee! Actually deliver the signal. */ handle_signal (&ksig, regs);return ;
include/linux/entry-common.h +5 -1
@@ -37,6 +37,10 @@# define _TIF_UPROBE (0)#endif +#ifndef _TIF_NOTIFY_SIGNAL +# define _TIF_NOTIFY_SIGNAL (0) +#endif +/* * TIF flags handled in syscall_enter_from_usermode() */
@@ -69,7 +73,7 @@#define EXIT_TO_USER_MODE_WORK \ (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ - _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | \ + _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ ARCH_EXIT_TO_USER_MODE_WORK)/**
include/linux/entry-kvm.h +2 -2
@@ -11,8 +11,8 @@# define ARCH_XFER_TO_GUEST_MODE_WORK (0)#endif -#define XFER_TO_GUEST_MODE_WORK \ - (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ +#define XFER_TO_GUEST_MODE_WORK \ + (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL | \ _TIF_NOTIFY_RESUME | ARCH_XFER_TO_GUEST_MODE_WORK)struct kvm_vcpu ;
include/linux/sched/signal.h +10 -1
@@ -360,6 +360,15 @@ static inline int task_sigpending(struct task_struct *p)static inline int signal_pending (struct task_struct *p) { +#ifdef TIF_NOTIFY_SIGNAL +/* +* TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same +* behavior in terms of ensuring that we break out of wait loops +* so that notify signal callbacks can be processed. +*/ +if (unlikely (test_tsk_thread_flag (p,TIF_NOTIFY_SIGNAL ))) +return 1 ; +#endif return task_sigpending (p); }
@@ -507,7 +516,7 @@ extern int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize);static inline void restore_saved_sigmask_unless (bool interrupted) {if (interrupted) -WARN_ON (!test_thread_flag (TIF_SIGPENDING )); +WARN_ON (!signal_pending (current));else restore_saved_sigmask (); }
include/linux/tracehook.h +27 -0
@@ -198,4 +198,31 @@ static inline void tracehook_notify_resume(struct pt_regs *regs) blkcg_maybe_throttle_current(); } +/* +* called by exit_to_user_mode_loop() if ti_work & _TIF_NOTIFY_SIGNAL. This +* is currently used by TWA_SIGNAL based task_work, which requires breaking +* wait loops to ensure that task_work is noticed and run. +*/ +static inline void tracehook_notify_signal (void ) +{ +#ifdef TIF_NOTIFY_SIGNAL +clear_thread_flag (TIF_NOTIFY_SIGNAL ); +smp_mb__after_atomic (); +if (current->task_works) +task_work_run (); +#endif +} + +/* +* Called when we have work to process from exit_to_user_mode_loop() +*/ +static inline void set_notify_signal (struct task_struct *task) +{ +#ifdef TIF_NOTIFY_SIGNAL +if (!test_and_set_tsk_thread_flag (task,TIF_NOTIFY_SIGNAL ) && + !wake_up_state (task,TASK_INTERRUPTIBLE )) +kick_process (task); +#endif +} +#endif /* <linux/tracehook.h> */
kernel/entry/common.c +1 -1
@@ -157,7 +157,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,if (ti_work & _TIF_PATCH_PENDING)klp_update_patch_state (current); -if (ti_work & _TIF_SIGPENDING) +if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))arch_do_signal (regs);if (ti_work & _TIF_NOTIFY_RESUME) {
kernel/entry/kvm.c +3 -0
@@ -8,6 +8,9 @@ static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) do { int ret; + if (ti_work & _TIF_NOTIFY_SIGNAL) + tracehook_notify_signal(); + if (ti_work & _TIF_SIGPENDING) { kvm_handle_signal_exit(vcpu); return -EINTR;
Signature · 1 line
2.28.0
Oleg Nesterov
maintainerRe: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
Once again, I am fine with this patch, just a minor comment...
arch/x86/kernel/signal.c void arch_do_signal(struct pt_regs *regs) · line 814
Quoted 12 lines · Jens Axboe
arch/x86/kernel/signal.c +4
-1@@ -808,7 +808,10 @@ void arch_do_signal(struct pt_regs *regs) {struct ksignal ksig; -if (get_signal (&ksig)) { +if (test_thread_flag (TIF_NOTIFY_SIGNAL )) +tracehook_notify_signal (); + +if (task_sigpending (current) &&get_signal (&ksig)) {
I suggested to change arch_do_signal() because somehow I like it this way ;)
And because we can easily pass the "ti_work" mask to arch_do_signal() and avoid test_thread_flag/task_sigpending.
Hmm. I just noticed that only x86 uses arch_do_signal(), so perhaps you can add this change to this patch right now? Up to you.
On the other hand, we could add
if (test_thread_flag(TIF_NOTIFY_SIGNAL)) tracehook_notify_signal(); if (!task_sigpending(current)) return 0;
at the start of get_signal() instead. Somehow I don't really like this, but this way we do need less changes in arch-dependant code. Again, up to you.
Oleg.
Jens Axboe
maintainerRe: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
Quoted 18 lines · Oleg Nesterov
Once again, I am fine with this patch, just a minor comment...
arch/x86/kernel/signal.c void arch_do_signal(struct pt_regs *regs) · line 814
Quoted 12 lines · Jens Axboe
arch/x86/kernel/signal.c +4
-1@@ -808,7 +808,10 @@ void arch_do_signal(struct pt_regs *regs) {struct ksignal ksig; -if (get_signal (&ksig)) { +if (test_thread_flag (TIF_NOTIFY_SIGNAL )) +tracehook_notify_signal (); + +if (task_sigpending (current) &&get_signal (&ksig)) {I suggested to change
arch_do_signal()because somehow I like it this way ;)
Quoted 5 lines
And because we can easily pass the "
ti_work" mask toarch_do_signal()and avoid test_thread_flag/task_sigpending.Hmm. I just noticed that only x86 uses
arch_do_signal(), so perhaps you can add this change to this patch right now? Up to you.
Sure, we can do that. Incremental on top then looks like the below. I don't feel that strongly about it, but it is nice to avoid re-testing the flags.
arch/x86/kernel/signal.c +3 -3
@@ -804,14 +804,14 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)* want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. */ -void arch_do_signal (struct pt_regs *regs) +void arch_do_signal (struct pt_regs *regs,unsigned long ti_work) {struct ksignal ksig; -if (test_thread_flag (TIF_NOTIFY_SIGNAL )) +if (ti_work & _TIF_NOTIFY_SIGNAL)tracehook_notify_signal (); -if (task_sigpending (current) &&get_signal (&ksig)) { +if ((ti_work & _TIF_SIGPENDING) &&get_signal (&ksig)) {/* Whee! Actually deliver the signal. */ handle_signal (&ksig, regs);return ;
include/linux/entry-common.h +2 -1
@@ -265,10 +265,11 @@ static __always_inline void arch_exit_to_user_mode(void) { }/** * arch_do_signal - Architecture specific signal delivery function * @regs: Pointer to currents pt_regs +* @ti_work task thread info work flags * * Invoked from exit_to_user_mode_loop(). */ -void arch_do_signal (struct pt_regs *regs); +void arch_do_signal (struct pt_regs *regs,unsigned long ti_work);/** * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit()
kernel/entry/common.c +2 -2
@@ -135,7 +135,7 @@ static __always_inline void exit_to_user_mode(void) }/* Workaround to allow gradual conversion of architecture code */ -void __weak arch_do_signal (struct pt_regs *regs) { } +void __weak arch_do_signal (struct pt_regs *regs,unsigned long ti_work) { }static unsigned long exit_to_user_mode_loop (struct pt_regs *regs,unsigned long ti_work)
@@ -158,7 +158,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,klp_update_patch_state (current);if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) -arch_do_signal (regs); +arch_do_signal (regs, ti_work);if (ti_work & _TIF_NOTIFY_RESUME) {tracehook_notify_resume (regs);
Signature · 1 line
Jens Axboe
Re: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
Quoted 6 lines · Jens Axboe
Quoted 2 lines
Hmm. I just noticed that only x86 uses
arch_do_signal(), so perhaps you can add this change to this patch right now? Up to you.Sure, we can do that. Incremental on top then looks like the below. I don't feel that strongly about it, but it is nice to avoid re-testing the flags.
Yes it makes sense. Can you please make the signature change of arch_do_signal() in a preparatory patch and only make use of it when you add the TIF bit to x86?
Thanks,
tglx
Jens Axboe
maintainerRe: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
Quoted 11 lines · Thomas Gleixner
Quoted 6 lines · Jens Axboe
Quoted 2 lines
Hmm. I just noticed that only x86 uses
arch_do_signal(), so perhaps you can add this change to this patch right now? Up to you.Sure, we can do that. Incremental on top then looks like the below. I don't feel that strongly about it, but it is nice to avoid re-testing the flags.
Yes it makes sense. Can you please make the signature change of
arch_do_signal()in a preparatory patch and only make use of it when you add the TIF bit to x86?
Ala:
with the pre generic patch being:
Signature · 1 line
Jens Axboe
Re: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
In reply to Jens AxboeQuoted 8 lines · Jens Axboe
This adds
TIF_NOTIFY_SIGNALhandling in the generic code, which if set, will return true ifsignal_pending()is used in a wait loop. That causes an exit of the loop so thatnotify_signaltracehooks can be run. If the wait loop is currently inside a system call, the system call is restarted oncetask_workhas been processed.x86 is using the generic entry code, add the necessary
TIF_NOTIFY_SIGNALdefinitions for it.
Can you please split that into core changes and a patch which adds support for x86?
Quoted 3 lines
static inline int
signal_pending(struct task_struct *p){
+#ifdefTIF_NOTIFY_SIGNAL
As I said in the other thread, plase make this
#if defined(CONFIG_GENERIC_ENTRY) && defined(TIF_NOTIFY_SIGNAL)
include/linux/tracehook.h static inline void tracehook_notify_resume(struct pt_regs *regs) · line 208
Quoted 8 lines
include/linux/tracehook.h +8
-0+/* +* called by exit_to_user_mode_loop() if ti_work & _TIF_NOTIFY_SIGNAL. This +* is currently used by TWA_SIGNAL based task_work, which requires breaking +* wait loops to ensure that task_work is noticed and run. +*/ +static inline void tracehook_notify_signal (void ) +{ +#ifdef TIF_NOTIFY_SIGNAL
Ditto.
include/linux/tracehook.h static inline void tracehook_notify_resume(struct pt_regs *regs) · line 221
Quoted 13 lines
include/linux/tracehook.h +13
-0+clear_thread_flag (TIF_NOTIFY_SIGNAL ); +smp_mb__after_atomic (); +if (current->task_works) +task_work_run (); +#endif +} + +/* +* Called when we have work to process from exit_to_user_mode_loop() +*/ +static inline void set_notify_signal (struct task_struct *task) +{ +#ifdef TIF_NOTIFY_SIGNAL
And this one.
Other than that, this approach of using arch_do_signal() addresses my earlier concerns about the syscall restart machinery.
Thanks,
tglx
Jens Axboe
maintainerRe: [PATCH 3/4] kernel: add support for TIF_NOTIFY_SIGNAL
Quoted 7 lines · Thomas Gleixner
Quoted 6 lines · Jens Axboe
This adds
TIF_NOTIFY_SIGNALhandling in the generic code, which if set, will return true ifsignal_pending()is used in a wait loop. That causes an exit of the loop so thatnotify_signaltracehooks can be run. If the wait loop is currently inside a system call, the system call is restarted oncetask_workhas been processed.
Quoted 5 lines
Quoted 2 lines
x86 is using the generic entry code, add the necessary
TIF_NOTIFY_SIGNALdefinitions for it.Can you please split that into core changes and a patch which adds support for x86?
Sure, I'll split it into generic and then x86 after that.
Quoted 7 lines
Quoted 3 lines
static inline int
signal_pending(struct task_struct *p){
+#ifdefTIF_NOTIFY_SIGNALAs I said in the other thread, plase make this
#if
defined(CONFIG_GENERIC_ENTRY)&&defined(TIF_NOTIFY_SIGNAL)
Thanks, I'll make those tweaks.
Quoted 2 lines
Other than that, this approach of using
arch_do_signal()addresses my earlier concerns about the syscall restart machinery.
Great! Thanks for taking a look.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 4/4 task_work: use TIF_NOTIFY_SIGNAL if available
In reply to Jens AxboeIf the arch supports TIF_NOTIFY_SIGNAL, then use that for TWA_SIGNAL as it's more efficient than using the signal delivery method. This is especially true on threaded applications, where ->sighand is shared across threads, but it's also lighter weight on non-shared cases.
io_uring is a heavy consumer of TWA_SIGNAL based task_work. On my test box, even just using 16 threads shows a nice improvement running an io_uring based echo server.
stock kernel:
0.01% <= 0.1 milliseconds
95.86% <= 0.2 milliseconds
98.27% <= 0.3 milliseconds
99.71% <= 0.4 milliseconds
100.00% <= 0.5 milliseconds
100.00% <= 0.6 milliseconds
100.00% <= 0.7 milliseconds
100.00% <= 0.8 milliseconds
100.00% <= 0.9 milliseconds
100.00% <= 1.0 milliseconds
100.00% <= 1.1 milliseconds
100.00% <= 2 milliseconds
100.00% <= 3 milliseconds
100.00% <= 3 milliseconds
1378930.00 requests per second
~1600% CPU
1.38M requests/second, and all 16 CPUs are maxed out.
patched kernel:
0.01% <= 0.1 milliseconds
98.24% <= 0.2 milliseconds
99.47% <= 0.3 milliseconds
99.99% <= 0.4 milliseconds
100.00% <= 0.5 milliseconds
100.00% <= 0.6 milliseconds
100.00% <= 0.7 milliseconds
100.00% <= 0.8 milliseconds
100.00% <= 0.9 milliseconds
100.00% <= 1.2 milliseconds
1666111.38 requests per second
~1450% CPU
1.67M requests/second, and we're no longer just hammering on the sighand lock. The original reporter states:
"For 5.7.15 my benchmark achieves 1.6M qps and system cpu is at ~80%.
for 5.7.16 or later it achieves only 1M qps and the system cpu is is at ~100%"
with the only difference there being that TWA_SIGNAL is used unconditionally in 5.7.16, since we need it to be able to solve an inability to run task_work if the application is waiting in the kernel already on an event that needs task_work run to be satisfied. Also see commit 0ba9c9edcd15.
--- kernel/task_work.c | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-)
kernel/task_work.c +29 -12
@@ -5,6 +5,34 @@static struct callback_head work_exited;/* all we need is ->next == NULL */ +/* +* TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster +* than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is +* shared for threads, and can cause contention on sighand->lock. Even for +* the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking +* or IRQ disabling is involved for notification (or running) purposes. +*/ +static void task_work_notify_signal (struct task_struct *task) +{ +#ifdef TIF_NOTIFY_SIGNAL +set_notify_signal (task); +#else +unsigned long flags; + +/* +* Only grab the sighand lock if we don't already have some +* task_work pending. This pairs with the smp_store_mb() +* in get_signal(), see comment there. +*/ +if (!(READ_ONCE (task->jobctl) &JOBCTL_TASK_WORK ) && +lock_task_sighand (task, &flags)) { + task->jobctl |=JOBCTL_TASK_WORK ; +signal_wake_up (task,0 ); +unlock_task_sighand (task, &flags); + } +#endif +} +/** * task_work_add - ask the @task to execute @work->func() * @task: the task which should run the callback
@@ -28,7 +56,6 @@ int task_work_add(struct task_struct *task, struct callback_head *work, int notify) { struct callback_head *head; -unsigned long flags; do { head = READ_ONCE(task->task_works);
@@ -42,17 +69,7 @@ task_work_add(struct task_struct *task, struct callback_head *work, int notify)set_notify_resume (task);break ;case TWA_SIGNAL : -/* -* Only grab the sighand lock if we don't already have some -* task_work pending. This pairs with the smp_store_mb() -* in get_signal(), see comment there. -*/ -if (!(READ_ONCE (task->jobctl) &JOBCTL_TASK_WORK ) && -lock_task_sighand (task, &flags)) { - task->jobctl |=JOBCTL_TASK_WORK ; -signal_wake_up (task,0 ); -unlock_task_sighand (task, &flags); - } +task_work_notify_signal (task);break ; }
Signature · 1 line
2.28.0
Re: [PATCH 4/4] task_work: use TIF_NOTIFY_SIGNAL if available
Quoted 16 lines · Jens Axboe
kernel/task_work.c +16
-0+/* +* TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster +* than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is +* shared for threads, and can cause contention on sighand->lock. Even for +* the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking +* or IRQ disabling is involved for notification (or running) purposes. +*/ +static void task_work_notify_signal (struct task_struct *task) +{ +#ifdef TIF_NOTIFY_SIGNAL +set_notify_signal (task); +#else +unsigned long flags; + +/* +* Only grab the sighand lock if we don't already have some
kernel/task_work.c line 33
Quoted 10 lines
kernel/task_work.c +10
-0+* task_work pending. This pairs with the smp_store_mb() +* in get_signal(), see comment there. +*/ +if (!(READ_ONCE (task->jobctl) &JOBCTL_TASK_WORK ) && +lock_task_sighand (task, &flags)) { + task->jobctl |=JOBCTL_TASK_WORK ; +signal_wake_up (task,0 ); +unlock_task_sighand (task, &flags); + } +#endif
Same #ifdeffery comment as before.
Thanks,
tglx
Jens Axboe
maintainerRe: [PATCH 4/4] task_work: use TIF_NOTIFY_SIGNAL if available
Quoted 21 lines · Thomas Gleixner
Quoted 20 lines · Jens Axboe
kernel/task_work.c +20
-0+/* +* TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster +* than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is +* shared for threads, and can cause contention on sighand->lock. Even for +* the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking +* or IRQ disabling is involved for notification (or running) purposes. +*/ +static void task_work_notify_signal (struct task_struct *task) +{ +#ifdef TIF_NOTIFY_SIGNAL +set_notify_signal (task); +#else +unsigned long flags; + +/* +* Only grab the sighand lock if we don't already have some +* task_work pending. This pairs with the smp_store_mb() +* in get_signal(), see comment there. +*/ + if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) &&
Quoted 8 lines
kernel/task_work.c line 33
Quoted 6 lines
kernel/task_work.c +6
-0+ lock_task_sighand(task, &flags)) { + task->jobctl |=JOBCTL_TASK_WORK ; +signal_wake_up (task,0 ); +unlock_task_sighand (task, &flags); + } +#endif Same #ifdeffery comment as before.
Fixed up.
Signature · 1 line
Jens Axboe
Quoted 9 lines · Jens Axboe
Changes since v3:
- Drop not needed
io_uringchange
- Drop syscall restart split, handleTIF_NOTIFY_SIGNALfrom the arch signal handling, usingtask_sigpending()to see if we need to care about real signals.
- Fix a few over-zelaoustask_sigpending()changes
- CleanupWARN_ON()inrestore_saved_sigmask_unless()
but let me comment 3/4...
Jens Axboe
maintainerQuoted 4 lines · Oleg Nesterov
Quoted 3 lines · Jens Axboe
Changes since v3:
Quoted 8 lines
Quoted 6 lines
- Drop not needed
io_uringchange
- Drop syscall restart split, handleTIF_NOTIFY_SIGNALfrom the arch signal handling, usingtask_sigpending()to see if we need to care about real signals.
- Fix a few over-zelaoustask_sigpending()changes
- CleanupWARN_ON()inrestore_saved_sigmask_unless()Reviewed-by Oleg Nesterov <[email protected]>
Thanks, added.
Quoted 1 line
but let me comment 3/4...
Updated for that one too, this is the current patch:
Signature · 1 line
Jens Axboe