kprobes: Use dedicated kthread for kprobe optimizer
Masami Hiramatsu (Google)
maintainerPATCH v4 1/1 kprobes: Use dedicated kthread for kprobe optimizer
From: Masami Hiramatsu (Google) <[email protected]>
Instead of using generic workqueue, use a dedicated kthread for optimizing kprobes, because it can wait (sleep) for a long time inside the process by synchronize_rcu_task(). This means other works can be stopped until it finishes.
Suggested-by Steven Rostedt <[email protected]>
Signed-off-by Masami Hiramatsu (Google) <[email protected]>
---
Changes in v4:
- Update onto probes/for-next.(fix a conflict)
Changes in v3:
- Wait events in interruptible state for avoiding hung_task.
Changes in v2:
- Make the kthread unfreezable same as workqueue.
- Add kthread_should_stop() check right before calling optimizer too.
- Initialize optimizer_state to OPTIMIZER_ST_IDLE instead of 0.
--- kernel/kprobes.c | 106 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 20 deletions(-)
kernel/kprobes.c +86 -20
@@ -32,6 +32,7 @@#include <linux/debugfs.h> #include <linux/sysctl.h> #include <linux/kdebug.h> +#include <linux/kthread.h> #include <linux/memory.h> #include <linux/ftrace.h> #include <linux/cpu.h>
@@ -40,6 +41,7 @@#include <linux/perf_event.h> #include <linux/execmem.h> #include <linux/cleanup.h> +#include <linux/wait.h> #include <asm/sections.h> #include <asm/cacheflush.h>
@@ -514,9 +516,18 @@ static LIST_HEAD(optimizing_list);static LIST_HEAD (unoptimizing_list);static LIST_HEAD (freeing_list); -static void kprobe_optimizer (struct work_struct *work); -static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);static void optimize_kprobe (struct kprobe *p); +static struct task_struct *kprobe_optimizer_task; +static wait_queue_head_t kprobe_optimizer_wait; +static atomic_t optimizer_state; +enum { +OPTIMIZER_ST_IDLE =0 , +OPTIMIZER_ST_KICKED =1 , +OPTIMIZER_ST_FLUSHING =2 , +}; + +static DECLARE_COMPLETION (optimizer_completion); +#define OPTIMIZE_DELAY 5/*
@@ -609,14 +620,10 @@ static void do_free_cleaned_kprobes(void) } } -/* Start optimizer after OPTIMIZE_DELAY passed */ -static void kick_kprobe_optimizer (void ) -{ -schedule_delayed_work (&optimizing_work,OPTIMIZE_DELAY ); -} +static void kick_kprobe_optimizer (void );/* Kprobe jump optimizer */ -static void kprobe_optimizer (struct work_struct *work) +static void kprobe_optimizer (void ) {guard (mutex)(&kprobe_mutex);
@@ -647,9 +654,53 @@ static void kprobe_optimizer(struct work_struct *work)do_free_cleaned_kprobes (); } -/* Step 5: Kick optimizer again if needed */ +/* Step 5: Kick optimizer again if needed. But if there is a flush requested, */ +if (completion_done (&optimizer_completion)) +complete (&optimizer_completion); +if (!list_empty (&optimizing_list) || !list_empty (&unoptimizing_list)) -kick_kprobe_optimizer (); +kick_kprobe_optimizer ();/*normal kick*/ +} + +static int kprobe_optimizer_thread (void *data) +{ +while (!kthread_should_stop ()) { +/* To avoid hung_task, wait in interruptible state. */ +wait_event_interruptible (kprobe_optimizer_wait, +atomic_read (&optimizer_state) !=OPTIMIZER_ST_IDLE || +kthread_should_stop ()); + +if (kthread_should_stop ()) +break ; + +/* +* If it was a normal kick, wait for OPTIMIZE_DELAY. +* This wait can be interrupted by a flush request. +*/ +if (atomic_read (&optimizer_state) ==1 ) +wait_event_interruptible_timeout ( + kprobe_optimizer_wait, +atomic_read (&optimizer_state) ==OPTIMIZER_ST_FLUSHING || +kthread_should_stop (), +OPTIMIZE_DELAY ); + +if (kthread_should_stop ()) +break ; + +atomic_set (&optimizer_state,OPTIMIZER_ST_IDLE ); + +kprobe_optimizer (); + } +return 0 ; +} + +/* Start optimizer after OPTIMIZE_DELAY passed */ +static void kick_kprobe_optimizer (void ) +{ +lockdep_assert_held (&kprobe_mutex); +if (atomic_cmpxchg (&optimizer_state, +OPTIMIZER_ST_IDLE ,OPTIMIZER_ST_KICKED ) ==OPTIMIZER_ST_IDLE ) +wake_up (&kprobe_optimizer_wait); }static void wait_for_kprobe_optimizer_locked (void )
@@ -657,13 +708,17 @@ static void wait_for_kprobe_optimizer_locked(void)lockdep_assert_held (&kprobe_mutex);while (!list_empty (&optimizing_list) || !list_empty (&unoptimizing_list)) { -mutex_unlock (&kprobe_mutex); - -/* This will also make 'optimizing_work' execute immmediately */ -flush_delayed_work (&optimizing_work); -/* 'optimizing_work' might not have been queued yet, relax */ -cpu_relax (); +init_completion (&optimizer_completion); +/* +* Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's +* idle. If it's already kicked, it will see the state change. +*/ +if (atomic_xchg_acquire (&optimizer_state, +OPTIMIZER_ST_FLUSHING ) !=OPTIMIZER_ST_FLUSHING ) +wake_up (&kprobe_optimizer_wait); +mutex_unlock (&kprobe_mutex); +wait_for_completion (&optimizer_completion);mutex_lock (&kprobe_mutex); } }
@@ -1016,8 +1071,21 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) } } +static void __init init_optprobe (void ) +{ +#ifdef __ARCH_WANT_KPROBES_INSN_SLOT +/* Init 'kprobe_optinsn_slots' for allocation */ + kprobe_optinsn_slots.insn_size =MAX_OPTINSN_SIZE ; +#endif + +init_waitqueue_head (&kprobe_optimizer_wait); +atomic_set (&optimizer_state,OPTIMIZER_ST_IDLE ); + kprobe_optimizer_task =kthread_run (kprobe_optimizer_thread,NULL , +"kprobe-optimizer" ); +}#else /* !CONFIG_OPTPROBES */ +#define init_optprobe () do {} while (0)#define optimize_kprobe (p) do {} while (0)#define unoptimize_kprobe (p, f) do {} while (0)#define kill_optimized_kprobe (p) do {} while (0)
@@ -2700,10 +2768,8 @@ static int __init init_kprobes(void)/* By default, kprobes are armed */ kprobes_all_disarmed = false; -#if defined(CONFIG_OPTPROBES ) && defined(__ARCH_WANT_KPROBES_INSN_SLOT) -/* Init 'kprobe_optinsn_slots' for allocation */ - kprobe_optinsn_slots.insn_size =MAX_OPTINSN_SIZE ; -#endif +/* Initialize the optimization infrastructure */ +init_optprobe (); err =arch_init_kprobes ();if (!err)