rcu: Fix expedited GP deadlock
Frederic Weisbecker
maintainercover rcu: Fix expedited GP deadlock
TREE04 can trigger a writer stall if run with memory pressure. This is due to a circular dependency between waiting for expedited grace period and polling on expedited grace period when workqueues go back to mayday serialization.
Here is a proposal fix.
Changes since v2 (no functional changes, just renames and reorganization):
_ Move nocb cleanups to their own series
_ Rename can_queue parameter to use_worker [2/7]
_ Better explain the rename of the mutex [3/7]
_ New commit with just code move to ease review [4/7] _ Comment declaration of the new rnp->exp_worker field [5/7]
Thanks.
Frederic Weisbecker (7):
rcu/exp: Fix RCU expedited parallel grace period kworker allocation
failure recovery
rcu/exp: Handle RCU expedited grace period kworker allocation failure
rcu: s/boost_kthread_mutex/kthread_mutex
rcu/exp: Move expedited kthread worker creation functions above
rcutree_prepare_cpu()
rcu/exp: Make parallel exp gp kworker per rcu node rcu/exp: Handle parallel exp gp kworkers affinity
rcu/exp: Remove rcu_par_gp_wq
kernel/rcu/rcu.h | 5 -- kernel/rcu/tree.c | 175 +++++++++++++++++++++++++++------------ kernel/rcu/tree.h | 11 ++- kernel/rcu/tree_exp.h | 78 +++-------------- kernel/rcu/tree_plugin.h | 52 ++---------- 5 files changed, 142 insertions(+), 179 deletions(-)
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 1/7 rcu/exp: Fix RCU expedited parallel grace period kworker allocation failure recovery
Under CONFIG_RCU_EXP_KTHREAD=y, the nodes initialization for expedited grace periods is queued to a kworker. However if the allocation of that kworker failed, the nodes initialization is performed synchronously by the caller instead.
Now the check for kworker initialization failure relies on the kworker pointer to be NULL while its value might actually encapsulate an allocation failure error.
Make sure to handle this case.
--- kernel/rcu/tree.c | 1 + 1 file changed, 1 insertion(+)
kernel/rcu/tree.c +1 -0
@@ -4760,6 +4760,7 @@ static void __init rcu_start_exp_gp_kworkers(void) rcu_exp_par_gp_kworker =kthread_create_worker (0 , par_gp_kworker_name);if (IS_ERR_OR_NULL (rcu_exp_par_gp_kworker)) {pr_err ("Failed to create %s!\n" , par_gp_kworker_name); + rcu_exp_par_gp_kworker =NULL ;kthread_destroy_worker (rcu_exp_gp_kworker);return ; }
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 2/7 rcu/exp: Handle RCU expedited grace period kworker allocation failure
In reply to Frederic WeisbeckerJust like is done for the kworker performing nodes initialization, gracefully handle the possible allocation failure of the RCU expedited grace period main kworker.
While at it perform a rename of the related checking functions to better reflect the expedited specifics.
--- kernel/rcu/tree.c | 2 ++ kernel/rcu/tree_exp.h | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-)
kernel/rcu/tree.c +2 -0
@@ -4754,6 +4754,7 @@ static void __init rcu_start_exp_gp_kworkers(void) rcu_exp_gp_kworker =kthread_create_worker (0 , gp_kworker_name);if (IS_ERR_OR_NULL (rcu_exp_gp_kworker)) {pr_err ("Failed to create %s!\n" , gp_kworker_name); + rcu_exp_gp_kworker =NULL ;return ; }
@@ -4762,6 +4763,7 @@ static void __init rcu_start_exp_gp_kworkers(void)pr_err ("Failed to create %s!\n" , par_gp_kworker_name); rcu_exp_par_gp_kworker =NULL ;kthread_destroy_worker (rcu_exp_gp_kworker); + rcu_exp_gp_kworker =NULL ;return ; }
kernel/rcu/tree_exp.h +19 -6
@@ -427,7 +427,12 @@ static void sync_rcu_exp_select_node_cpus(struct kthread_work *wp)__sync_rcu_exp_select_node_cpus (rewp); } -static inline bool rcu_gp_par_worker_started (void ) +static inline bool rcu_exp_worker_started (void ) +{ +return !!READ_ONCE (rcu_exp_gp_kworker); +} + +static inline bool rcu_exp_par_worker_started (void ) {return !!READ_ONCE (rcu_exp_par_gp_kworker); }
@@ -477,7 +482,12 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp)__sync_rcu_exp_select_node_cpus (rewp); } -static inline bool rcu_gp_par_worker_started (void ) +static inline bool rcu_exp_worker_started (void ) +{ +return !!READ_ONCE (rcu_gp_wq); +} + +static inline bool rcu_exp_par_worker_started (void ) {return !!READ_ONCE (rcu_par_gp_wq); }
@@ -540,7 +550,7 @@ static void sync_rcu_exp_select_cpus(void) rnp->exp_need_flush = false;if (!READ_ONCE (rnp->expmask))continue ;/* Avoid early boot non-existent wq. */ -if (!rcu_gp_par_worker_started () || +if (!rcu_exp_par_worker_started () || rcu_scheduler_active !=RCU_SCHEDULER_RUNNING ||rcu_is_last_leaf_node (rnp)) {/* No worker started yet or last leaf, do direct call. */
@@ -955,7 +965,7 @@ static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp)*/ void synchronize_rcu_expedited (void ) { -bool boottime = (rcu_scheduler_active ==RCU_SCHEDULER_INIT ); +bool use_worker;unsigned long flags;struct rcu_exp_work rew;struct rcu_node *rnp;
@@ -966,6 +976,9 @@ void synchronize_rcu_expedited(void)lock_is_held (&rcu_sched_lock_map), "Illegal synchronize_rcu_expedited() in RCU read-side critical section"); + use_worker = (rcu_scheduler_active !=RCU_SCHEDULER_INIT ) && +rcu_exp_worker_started (); +/* Is the state is such that the call is a grace period? */ if (rcu_blocking_is_gp ()) {// Note well that this code runs with !PREEMPT && !SMP.
@@ -995,7 +1008,7 @@ void synchronize_rcu_expedited(void)return ;/* Someone else did our work for us. */ /* Ensure that load happens before action based on it. */ -if (unlikely (boottime)) { +if (unlikely (!use_worker)) {/* Direct call during scheduler init and early_initcalls(). */ rcu_exp_sel_wait_wake (s); }else {
@@ -1013,7 +1026,7 @@ void synchronize_rcu_expedited(void)/* Let the next expedited grace period start. */ mutex_unlock (&rcu_state.exp_mutex); -if (likely (!boottime)) +if (likely (use_worker))synchronize_rcu_expedited_destroy_work (&rew); }EXPORT_SYMBOL_GPL (synchronize_rcu_expedited);
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 3/7 rcu: s/boost_kthread_mutex/kthread_mutex
In reply to Frederic WeisbeckerThis mutex is currently protecting per node boost kthreads creation and affinity setting across CPU hotplug operations.
Since the expedited kworkers will soon be split per node as well, they will be subject to the same concurrency constraints against hotplug.
Therefore their creation and affinity tuning operations will be grouped with those of boost kthreads and then rely on the same mutex.
To prepare for that, generalize its name.
--- kernel/rcu/tree.c | 2 +- kernel/rcu/tree.h | 2 +- kernel/rcu/tree_plugin.h | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-)
kernel/rcu/tree.c +1 -1
@@ -4929,7 +4929,7 @@ static void __init rcu_init_one(void)init_waitqueue_head (&rnp->exp_wq[2 ]);init_waitqueue_head (&rnp->exp_wq[3 ]);spin_lock_init (&rnp->exp_lock); -mutex_init (&rnp->boost_kthread_mutex); +mutex_init (&rnp->kthread_mutex);raw_spin_lock_init (&rnp->exp_poll_lock); rnp->exp_seq_poll_rq =RCU_GET_STATE_COMPLETED ;INIT_WORK (&rnp->exp_poll_wq, sync_rcu_do_polled_gp);
kernel/rcu/tree.h +1 -1
@@ -113,7 +113,7 @@ struct rcu_node {/* side effect, not as a lock. */ unsigned long boost_time;/* When to start boosting (jiffies). */ -struct mutex boost_kthread_mutex; +struct mutex kthread_mutex;/* Exclusion for thread spawning and affinity */ /* manipulation. */ struct task_struct *boost_kthread_task;
kernel/rcu/tree_plugin.h +5 -5
@@ -1195,7 +1195,7 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)struct sched_param sp;struct task_struct *t; -mutex_lock (&rnp->boost_kthread_mutex); +mutex_lock (&rnp->kthread_mutex);if (rnp->boost_kthread_task || !rcu_scheduler_fully_active) goto out;
@@ -1212,7 +1212,7 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)wake_up_process (t);/* get to TASK_INTERRUPTIBLE quickly. */ out: -mutex_unlock (&rnp->boost_kthread_mutex); +mutex_unlock (&rnp->kthread_mutex); }/*
@@ -1224,7 +1224,7 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)* no outgoing CPU. If there are no CPUs left in the affinity set, * this function allows the kthread to execute on any CPU. * -* Any future concurrent calls are serialized via ->boost_kthread_mutex. +* Any future concurrent calls are serialized via ->kthread_mutex. */ static void rcu_boost_kthread_setaffinity (struct rcu_node *rnp,int outgoingcpu) {
@@ -1237,7 +1237,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)return ;if (!zalloc_cpumask_var (&cm,GFP_KERNEL ))return ; -mutex_lock (&rnp->boost_kthread_mutex); +mutex_lock (&rnp->kthread_mutex); mask =rcu_rnp_online_cpus (rnp);for_each_leaf_node_possible_cpu (rnp, cpu) if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
@@ -1250,7 +1250,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)cpumask_clear_cpu (outgoingcpu, cm); }set_cpus_allowed_ptr (t, cm); -mutex_unlock (&rnp->boost_kthread_mutex); +mutex_unlock (&rnp->kthread_mutex);free_cpumask_var (cm); }
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 4/7 rcu/exp: Move expedited kthread worker creation functions above rcutree_prepare_cpu()
In reply to Frederic WeisbeckerThe expedited kthread worker performing the per node initialization is going to be split into per node kthreads. As such, the future per node kthread creation will need to be called from CPU hotplug callbacks instead of an initcall, right beside the per node boost kthread creation.
To prepare for that, move the kthread worker creation above rcutree_prepare_cpu() as a first step to make the review smoother for the upcoming modifications.
No intended functional change.
--- kernel/rcu/tree.c | 96 +++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 48 deletions(-)
kernel/rcu/tree.c +48 -48
@@ -4405,6 +4405,54 @@ rcu_boot_init_percpu_data(int cpu)rcu_boot_init_nocb_percpu_data (rdp); } +#ifdef CONFIG_RCU_EXP_KTHREAD +struct kthread_worker *rcu_exp_gp_kworker; +struct kthread_worker *rcu_exp_par_gp_kworker; + +static void __init rcu_start_exp_gp_kworkers (void ) +{ +const char *par_gp_kworker_name ="rcu_exp_par_gp_kthread_worker" ; +const char *gp_kworker_name ="rcu_exp_gp_kthread_worker" ; +struct sched_param param = { .sched_priority = kthread_prio }; + + rcu_exp_gp_kworker =kthread_create_worker (0 , gp_kworker_name); +if (IS_ERR_OR_NULL (rcu_exp_gp_kworker)) { +pr_err ("Failed to create %s!\n" , gp_kworker_name); + rcu_exp_gp_kworker =NULL ; +return ; + } + + rcu_exp_par_gp_kworker =kthread_create_worker (0 , par_gp_kworker_name); +if (IS_ERR_OR_NULL (rcu_exp_par_gp_kworker)) { +pr_err ("Failed to create %s!\n" , par_gp_kworker_name); + rcu_exp_par_gp_kworker =NULL ; +kthread_destroy_worker (rcu_exp_gp_kworker); + rcu_exp_gp_kworker =NULL ; +return ; + } + +sched_setscheduler_nocheck (rcu_exp_gp_kworker->task,SCHED_FIFO , ¶m); +sched_setscheduler_nocheck (rcu_exp_par_gp_kworker->task,SCHED_FIFO , + ¶m); +} + +static inline void rcu_alloc_par_gp_wq (void ) +{ +} +#else /* !CONFIG_RCU_EXP_KTHREAD */ +struct workqueue_struct *rcu_par_gp_wq; + +static void __init rcu_start_exp_gp_kworkers (void ) +{ +} + +static inline void rcu_alloc_par_gp_wq (void ) +{ + rcu_par_gp_wq =alloc_workqueue ("rcu_par_gp" ,WQ_MEM_RECLAIM ,0 ); +WARN_ON (!rcu_par_gp_wq); +} +#endif /* CONFIG_RCU_EXP_KTHREAD */ +/* * Invoked early in the CPU-online process, when pretty much all services * are available. The incoming CPU is not present.
@@ -4741,54 +4789,6 @@ static int rcu_pm_notify(struct notifier_block *self,return NOTIFY_OK ; } -#ifdef CONFIG_RCU_EXP_KTHREAD -struct kthread_worker *rcu_exp_gp_kworker; -struct kthread_worker *rcu_exp_par_gp_kworker; - -static void __init rcu_start_exp_gp_kworkers (void ) -{ -const char *par_gp_kworker_name ="rcu_exp_par_gp_kthread_worker" ; -const char *gp_kworker_name ="rcu_exp_gp_kthread_worker" ; -struct sched_param param = { .sched_priority = kthread_prio }; - - rcu_exp_gp_kworker =kthread_create_worker (0 , gp_kworker_name); -if (IS_ERR_OR_NULL (rcu_exp_gp_kworker)) { -pr_err ("Failed to create %s!\n" , gp_kworker_name); - rcu_exp_gp_kworker =NULL ; -return ; - } - - rcu_exp_par_gp_kworker =kthread_create_worker (0 , par_gp_kworker_name); -if (IS_ERR_OR_NULL (rcu_exp_par_gp_kworker)) { -pr_err ("Failed to create %s!\n" , par_gp_kworker_name); - rcu_exp_par_gp_kworker =NULL ; -kthread_destroy_worker (rcu_exp_gp_kworker); - rcu_exp_gp_kworker =NULL ; -return ; - } - -sched_setscheduler_nocheck (rcu_exp_gp_kworker->task,SCHED_FIFO , ¶m); -sched_setscheduler_nocheck (rcu_exp_par_gp_kworker->task,SCHED_FIFO , - ¶m); -} - -static inline void rcu_alloc_par_gp_wq (void ) -{ -} -#else /* !CONFIG_RCU_EXP_KTHREAD */ -struct workqueue_struct *rcu_par_gp_wq; - -static void __init rcu_start_exp_gp_kworkers (void ) -{ -} - -static inline void rcu_alloc_par_gp_wq (void ) -{ - rcu_par_gp_wq =alloc_workqueue ("rcu_par_gp" ,WQ_MEM_RECLAIM ,0 ); -WARN_ON (!rcu_par_gp_wq); -} -#endif /* CONFIG_RCU_EXP_KTHREAD */ -/* * Spawn the kthreads that handle RCU's grace periods. */
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 5/7 rcu/exp: Make parallel exp gp kworker per rcu node
In reply to Frederic WeisbeckerWhen CONFIG_RCU_EXP_KTHREAD=n, the expedited grace period per node initialization is performed in parallel via workqueues (one work per node).
However in CONFIG_RCU_EXP_KTHREAD=y, this per node initialization is performed by a single kworker serializing each node initialization (one work for all nodes).
The second part is certainly less scalable and efficient beyond a single leaf node.
To improve this, expand this single kworker into per-node kworkers. This new layout is eventually intended to remove the workqueues based implementation since it will essentially now become duplicate code.
--- kernel/rcu/rcu.h | 1 - kernel/rcu/tree.c | 65 ++++++++++++++++++++++++++-------------- kernel/rcu/tree.h | 3 ++ kernel/rcu/tree_exp.h | 10 +++---- kernel/rcu/tree_plugin.h | 10 ++----- 5 files changed, 54 insertions(+), 35 deletions(-)
kernel/rcu/rcu.h +0 -1
@@ -625,7 +625,6 @@ void rcu_force_quiescent_state(void);extern struct workqueue_struct *rcu_gp_wq;#ifdef CONFIG_RCU_EXP_KTHREAD extern struct kthread_worker *rcu_exp_gp_kworker; -extern struct kthread_worker *rcu_exp_par_gp_kworker;#else /* !CONFIG_RCU_EXP_KTHREAD */ extern struct workqueue_struct *rcu_par_gp_wq;#endif /* CONFIG_RCU_EXP_KTHREAD */
kernel/rcu/tree.c +43 -22
106 lines not on this page · raw message
kernel/rcu/tree.h +3 -0
@@ -72,6 +72,9 @@ struct rcu_node {/* Online CPUs for next expedited GP. */ /* Any CPU that has ever been online will */ /* have its bit set. */ +struct kthread_worker *exp_kworker; +/* Workers performing per node expedited GP */ +/* initialization. */ unsigned long cbovldmask;/* CPUs experiencing callback overload. */ unsigned long ffmask;/* Fully functional CPUs. */
kernel/rcu/tree_exp.h +5 -5
@@ -432,9 +432,9 @@ static inline bool rcu_exp_worker_started(void)return !!READ_ONCE (rcu_exp_gp_kworker); } -static inline bool rcu_exp_par_worker_started (void ) +static inline bool rcu_exp_par_worker_started (struct rcu_node *rnp) { -return !!READ_ONCE (rcu_exp_par_gp_kworker); +return !!READ_ONCE (rnp->exp_kworker); }static inline void sync_rcu_exp_select_cpus_queue_work (struct rcu_node *rnp)
@@ -445,7 +445,7 @@ static inline void sync_rcu_exp_select_cpus_queue_work(struct rcu_node *rnp)* another work item on the same kthread worker can result in * deadlock. */ -kthread_queue_work (rcu_exp_par_gp_kworker, &rnp->rew.rew_work); +kthread_queue_work (READ_ONCE (rnp->exp_kworker), &rnp->rew.rew_work); }static inline void sync_rcu_exp_select_cpus_flush_work (struct rcu_node *rnp)
@@ -487,7 +487,7 @@ static inline bool rcu_exp_worker_started(void)return !!READ_ONCE (rcu_gp_wq); } -static inline bool rcu_exp_par_worker_started (void ) +static inline bool rcu_exp_par_worker_started (struct rcu_node *rnp) {return !!READ_ONCE (rcu_par_gp_wq); }
@@ -550,7 +550,7 @@ static void sync_rcu_exp_select_cpus(void) rnp->exp_need_flush = false;if (!READ_ONCE (rnp->expmask))continue ;/* Avoid early boot non-existent wq. */ -if (!rcu_exp_par_worker_started () || +if (!rcu_exp_par_worker_started (rnp) || rcu_scheduler_active !=RCU_SCHEDULER_RUNNING ||rcu_is_last_leaf_node (rnp)) {/* No worker started yet or last leaf, do direct call. */
kernel/rcu/tree_plugin.h +3 -7
@@ -1195,14 +1195,13 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)struct sched_param sp;struct task_struct *t; -mutex_lock (&rnp->kthread_mutex); -if (rnp->boost_kthread_task || !rcu_scheduler_fully_active) - goto out; +if (rnp->boost_kthread_task) +return ; t =kthread_create (rcu_boost_kthread, (void *)rnp,"rcub/%d" , rnp_index);if (WARN_ON_ONCE (IS_ERR (t))) - goto out; +return ;raw_spin_lock_irqsave_rcu_node (rnp, flags); rnp->boost_kthread_task = t;
@@ -1210,9 +1209,6 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) sp.sched_priority = kthread_prio;sched_setscheduler_nocheck (t,SCHED_FIFO , &sp);wake_up_process (t);/* get to TASK_INTERRUPTIBLE quickly. */ - - out: -mutex_unlock (&rnp->kthread_mutex); }/*
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 6/7 rcu/exp: Handle parallel exp gp kworkers affinity
In reply to Frederic WeisbeckerAffine the parallel expedited gp kworkers to their respective RCU node in order to make them close to the cache their are playing with.
This reuses the boost kthreads machinery that probe into CPU hotplug operations such that the kthreads become/stay affine to their respective node as soon/long as they contain online CPUs. Otherwise and if the current CPU going down was the last online on the leaf node, the related kthread is affine to the housekeeping CPUs.
In the long run, this affinity VS CPU hotplug operation game should probably be implemented at the generic kthread level.
--- kernel/rcu/tree.c | 79 +++++++++++++++++++++++++++++++++++++--- kernel/rcu/tree_plugin.h | 42 ++------------------- 2 files changed, 78 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +74 -5
@@ -145,7 +145,7 @@ static int rcu_scheduler_fully_active __read_mostly;static void rcu_report_qs_rnp (unsigned long mask,struct rcu_node *rnp,unsigned long gps,unsigned long flags); -static void rcu_boost_kthread_setaffinity (struct rcu_node *rnp,int outgoingcpu); +static struct task_struct *rcu_boost_task (struct rcu_node *rnp);static void invoke_rcu_core (void );static void rcu_report_exp_rdp (struct rcu_data *rdp);static void sync_sched_exp_online_cleanup (int cpu);
@@ -4428,6 +4428,16 @@ static void rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp)sched_setscheduler_nocheck (kworker->task,SCHED_FIFO , ¶m); } +static struct task_struct *rcu_exp_par_gp_task (struct rcu_node *rnp) +{ +struct kthread_worker *kworker =READ_ONCE (rnp->exp_kworker); + +if (!kworker) +return NULL ; + +return kworker->task; +} +static void __init rcu_start_exp_gp_kworker (void ) {const char *name ="rcu_exp_gp_kthread_worker" ;
@@ -4452,6 +4462,11 @@ static void rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp) { } +static struct task_struct *rcu_exp_par_gp_task (struct rcu_node *rnp) +{ +return NULL ; +} +static void __init rcu_start_exp_gp_kworker (void ) { }
@@ -4530,13 +4545,67 @@ int rcutree_prepare_cpu(unsigned int cpu) }/* -* Update RCU priority boot kthread affinity for CPU-hotplug changes. +* Update kthreads affinity during CPU-hotplug changes. +* +* Set the per-rcu_node kthread's affinity to cover all CPUs that are +* served by the rcu_node in question. The CPU hotplug lock is still +* held, so the value of rnp->qsmaskinit will be stable. +* +* We don't include outgoingcpu in the affinity set, use -1 if there is +* no outgoing CPU. If there are no CPUs left in the affinity set, +* this function allows the kthread to execute on any CPU. +* +* Any future concurrent calls are serialized via ->kthread_mutex. */ -static void rcutree_affinity_setting (unsigned int cpu,int outgoing) +static void rcutree_affinity_setting (unsigned int cpu,int outgoingcpu) { -struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); +cpumask_var_t cm; +unsigned long mask; +struct rcu_data *rdp; +struct rcu_node *rnp; +struct task_struct *task_boost, *task_exp; -rcu_boost_kthread_setaffinity (rdp->mynode, outgoing); +if (!IS_ENABLED (CONFIG_RCU_EXP_KTHREAD ) && !IS_ENABLED (CONFIG_RCU_BOOST )) +return ; + + rdp =per_cpu_ptr (&rcu_data, cpu); + rnp = rdp->mynode; + + task_boost =rcu_boost_task (rnp); + task_exp =rcu_exp_par_gp_task (rnp); + +/* +* If CPU is the boot one, those tasks are created later from early +* initcall since kthreadd must be created first. +*/ +if (!task_boost && !task_exp) +return ; + +if (!zalloc_cpumask_var (&cm,GFP_KERNEL )) +return ; + +mutex_lock (&rnp->kthread_mutex); + mask =rcu_rnp_online_cpus (rnp); +for_each_leaf_node_possible_cpu (rnp, cpu) +if ((mask &leaf_node_cpu_bit (rnp, cpu)) && + cpu != outgoingcpu) +cpumask_set_cpu (cpu, cm); +cpumask_and (cm, cm,housekeeping_cpumask (HK_TYPE_RCU )); +if (cpumask_empty (cm)) { +cpumask_copy (cm,housekeeping_cpumask (HK_TYPE_RCU )); +if (outgoingcpu >=0 ) +cpumask_clear_cpu (outgoingcpu, cm); + } + +if (task_exp) +set_cpus_allowed_ptr (task_exp, cm); + +if (task_boost) +set_cpus_allowed_ptr (task_boost, cm); + +mutex_unlock (&rnp->kthread_mutex); + +free_cpumask_var (cm); }/*
kernel/rcu/tree_plugin.h +4 -38
57 lines not on this page · raw message
Signature · 1 line
2.34.1
Frederic Weisbecker
maintainerPATCH 7/7 rcu/exp: Remove rcu_par_gp_wq
In reply to Frederic WeisbeckerTREE04 running on short iterations can produce writer stalls of the following kind:
??? Writer stall state RTWS_EXP_SYNC(4) g3968 f0x0 ->state 0x2 cpu 0
task:rcu_torture_wri state:D stack:14568 pid:83 ppid:2 flags:0x00004000
Call Trace: <TASK> __schedule+0x2de/0x850 ? trace_event_raw_event_rcu_exp_funnel_lock+0x6d/0xb0 schedule+0x4f/0x90 synchronize_rcu_expedited+0x430/0x670 ? __pfx_autoremove_wake_function+0x10/0x10 ? __pfx_synchronize_rcu_expedited+0x10/0x10 do_rtws_sync.constprop.0+0xde/0x230 rcu_torture_writer+0x4b4/0xcd0 ? __pfx_rcu_torture_writer+0x10/0x10 kthread+0xc7/0xf0 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x2f/0x50 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1b/0x30 </TASK>
Waiting for an expedited grace period and polling for an expedited grace period both are operations that internally rely on the same workqueue performing necessary asynchronous work.
However, a dependency chain is involved between those two operations, as depicted below:
====== CPU 0 ======= ====== CPU 1 =======
synchronize_rcu_expedited()
exp_funnel_lock()
mutex_lock(&rcu_state.exp_mutex);
start_poll_synchronize_rcu_expedited
queue_work(rcu_gp_wq, &rnp->exp_poll_wq);
synchronize_rcu_expedited_queue_work()
queue_work(rcu_gp_wq, &rew->rew_work);
wait_event() // A, wait for &rew->rew_work completion
mutex_unlock() // B
//======> switch to kworker
sync_rcu_do_polled_gp() {
synchronize_rcu_expedited()
exp_funnel_lock()
mutex_lock(&rcu_state.exp_mutex); // C, wait B
....
} // DSince workqueues are usually implemented on top of several kworkers handling the queue concurrently, the above situation wouldn't deadlock most of the time because A then doesn't depend on D. But in case of memory stress, a single kworker may end up handling alone all the works in a serialized way. In that case the above layout becomes a problem because A then waits for D, closing a circular dependency:
A -> D -> C -> B -> A
This however only happens when CONFIG_RCU_EXP_KTHREAD=n. Indeed synchronize_rcu_expedited() is otherwise implemented on top of a kthread worker while polling still relies on rcu_gp_wq workqueue, breaking the above circular dependency chain.
Fix this with making expedited grace period to always rely on kthread worker. The workqueue based implementation is essentially a duplicate anyway now that the per-node initialization is performed by per-node kthread workers.
Meanwhile the CONFIG_RCU_EXP_KTHREAD switch is still kept around to manage the scheduler policy of these kthread workers.
--- kernel/rcu/rcu.h | 4 --- kernel/rcu/tree.c | 40 ++++-------------------- kernel/rcu/tree.h | 6 +--- kernel/rcu/tree_exp.h | 73 +------------------------------------------ 4 files changed, 8 insertions(+), 115 deletions(-)
kernel/rcu/rcu.h +0 -4
@@ -623,11 +623,7 @@ int rcu_get_gp_kthreads_prio(void);void rcu_fwd_progress_check (unsigned long j);void rcu_force_quiescent_state (void );extern struct workqueue_struct *rcu_gp_wq; -#ifdef CONFIG_RCU_EXP_KTHREAD extern struct kthread_worker *rcu_exp_gp_kworker; -#else /* !CONFIG_RCU_EXP_KTHREAD */ -extern struct workqueue_struct *rcu_par_gp_wq; -#endif /* CONFIG_RCU_EXP_KTHREAD */ void rcu_gp_slow_register (atomic_t *rgssp);void rcu_gp_slow_unregister (atomic_t *rgssp);#endif /* #else #ifdef CONFIG_TINY_RCU */
kernel/rcu/tree.c +6 -34
@@ -4405,7 +4405,6 @@ rcu_boot_init_percpu_data(int cpu)rcu_boot_init_nocb_percpu_data (rdp); } -#ifdef CONFIG_RCU_EXP_KTHREAD struct kthread_worker *rcu_exp_gp_kworker;static void rcu_spawn_exp_par_gp_kworker (struct rcu_node *rnp)
@@ -4425,7 +4424,9 @@ static void rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp)return ; }WRITE_ONCE (rnp->exp_kworker, kworker); -sched_setscheduler_nocheck (kworker->task,SCHED_FIFO , ¶m); + +if (IS_ENABLED (CONFIG_RCU_EXP_KTHREAD )) +sched_setscheduler_nocheck (kworker->task,SCHED_FIFO , ¶m); }static struct task_struct *rcu_exp_par_gp_task (struct rcu_node *rnp)
@@ -4449,39 +4450,14 @@ static void __init rcu_start_exp_gp_kworker(void) rcu_exp_gp_kworker =NULL ;return ; } -sched_setscheduler_nocheck (rcu_exp_gp_kworker->task,SCHED_FIFO , ¶m); -} - -static inline void rcu_alloc_par_gp_wq (void ) -{ -} -#else /* !CONFIG_RCU_EXP_KTHREAD */ -struct workqueue_struct *rcu_par_gp_wq; - -static void rcu_spawn_exp_par_gp_kworker (struct rcu_node *rnp) -{ -} - -static struct task_struct *rcu_exp_par_gp_task (struct rcu_node *rnp) -{ -return NULL ; -} - -static void __init rcu_start_exp_gp_kworker (void ) -{ -} -static inline void rcu_alloc_par_gp_wq (void ) -{ - rcu_par_gp_wq =alloc_workqueue ("rcu_par_gp" ,WQ_MEM_RECLAIM ,0 ); -WARN_ON (!rcu_par_gp_wq); +if (IS_ENABLED (CONFIG_RCU_EXP_KTHREAD )) +sched_setscheduler_nocheck (rcu_exp_gp_kworker->task,SCHED_FIFO , ¶m); } -#endif /* CONFIG_RCU_EXP_KTHREAD */ static void rcu_spawn_rnp_kthreads (struct rcu_node *rnp) { -if ((IS_ENABLED (CONFIG_RCU_EXP_KTHREAD ) || -IS_ENABLED (CONFIG_RCU_BOOST )) && rcu_scheduler_fully_active) { +if (rcu_scheduler_fully_active) {mutex_lock (&rnp->kthread_mutex);rcu_spawn_one_boost_kthread (rnp);rcu_spawn_exp_par_gp_kworker (rnp);
@@ -4565,9 +4541,6 @@ static void rcutree_affinity_setting(unsigned int cpu, int outgoingcpu)struct rcu_node *rnp;struct task_struct *task_boost, *task_exp; -if (!IS_ENABLED (CONFIG_RCU_EXP_KTHREAD ) && !IS_ENABLED (CONFIG_RCU_BOOST )) -return ; - rdp =per_cpu_ptr (&rcu_data, cpu); rnp = rdp->mynode;
@@ -5256,7 +5229,6 @@ void __init rcu_init(void)/* Create workqueue for Tree SRCU and for expedited GPs. */ rcu_gp_wq =alloc_workqueue ("rcu_gp" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!rcu_gp_wq); -rcu_alloc_par_gp_wq ();/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */
kernel/rcu/tree.h +1 -5
@@ -21,14 +21,10 @@#include "rcu_segcblist.h" -/* Communicate arguments to a workqueue handler. */ +/* Communicate arguments to a kthread worker handler. */ struct rcu_exp_work {unsigned long rew_s; -#ifdef CONFIG_RCU_EXP_KTHREAD struct kthread_work rew_work; -#else -struct work_struct rew_work; -#endif /* CONFIG_RCU_EXP_KTHREAD */ };/* RCU's kthread states for tracing. */
kernel/rcu/tree_exp.h +1 -72
109 lines not on this page · raw message
Signature · 1 line
2.34.1
Paul E. McKenney
maintainerQuoted 9 lines · Frederic Weisbecker
TREE04 can trigger a writer stall if run with memory pressure. This is due to a circular dependency between waiting for expedited grace period and polling on expedited grace period when workqueues go back to mayday serialization.
Here is a proposal fix.
Changes since v2 (no functional changes, just renames and reorganization):
Quoted 7 lines
_ Move nocb cleanups to their own series
_ Renamecan_queueparameter touse_worker[2/7]
_ Better explain the rename of the mutex [3/7]
_ New commit with just code move to ease review [4/7] _ Comment declaration of the newrnp->exp_workerfield [5/7]Thanks.
Hearing no objections, queued for testing and further review, thank you!
Thanx, Paul
Quoted 22 lines
Frederic Weisbecker (7):
rcu/exp: Fix RCU expedited parallel grace period kworker allocation
failure recovery
rcu/exp: Handle RCU expedited grace period kworker allocation failure
rcu: s/boost_kthread_mutex/kthread_mutex
rcu/exp: Move expedited kthread worker creation functions above
rcutree_prepare_cpu()
rcu/exp: Make parallel exp gp kworker per rcu node rcu/exp: Handle parallel exp gp kworkers affinity
rcu/exp: Removercu_par_gp_wqkernel/rcu/rcu.h | 5 -- kernel/rcu/tree.c | 175 +++++++++++++++++++++++++++------------ kernel/rcu/tree.h | 11 ++- kernel/rcu/tree_exp.h | 78 +++-------------- kernel/rcu/tree_plugin.h | 52 ++---------- 5 files changed, 142 insertions(+), 179 deletions(-)Signature · 1 line
2.34.1