rcu: Convert rcu_state.ofl_lock to raw_spinlock_t
PATCH 1/1 rcu: Convert rcu_state.ofl_lock to raw_spinlock_t
1e64b15a4b10 ("rcu: Fix grace-period hangs due to race with CPU offline") added spinlock_t ofl_lock to the rcu_state structure, then takes it with preemption disabled during CPU offline, giving RT sleeping lock heartburn.
Convert it to raw_spinlock_t.
--- kernel/rcu/tree.c | 12 ++++++------ kernel/rcu/tree.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-)
kernel/rcu/tree.c +6 -6
@@ -111,7 +111,7 @@ struct rcu_state sname##_state = { \ .abbr = sabbr, \ .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \ .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \ - .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \ + .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \ }RCU_STATE_INITIALIZER (rcu_sched,'s' , call_rcu_sched);
@@ -1962,13 +1962,13 @@ static bool rcu_gp_init(struct rcu_state*/ rsp->gp_state =RCU_GP_ONOFF ;rcu_for_each_leaf_node (rsp, rnp) { -spin_lock (&rsp->ofl_lock); +raw_spin_lock (&rsp->ofl_lock);raw_spin_lock_irq_rcu_node (rnp);if (rnp->qsmaskinit == rnp->qsmaskinitnext && !rnp->wait_blkd_tasks) {/* Nothing to do on this leaf rcu_node structure. */ raw_spin_unlock_irq_rcu_node (rnp); -spin_unlock (&rsp->ofl_lock); +raw_spin_unlock (&rsp->ofl_lock);continue ; }
@@ -2004,7 +2004,7 @@ static bool rcu_gp_init(struct rcu_state }raw_spin_unlock_irq_rcu_node (rnp); -spin_unlock (&rsp->ofl_lock); +raw_spin_unlock (&rsp->ofl_lock); }rcu_gp_slow (rsp, gp_preinit_delay);/* Races with CPU hotplug. */
@@ -3892,7 +3892,7 @@ static void rcu_cleanup_dying_idle_cpu(i/* Remove outgoing CPU from mask in the leaf rcu_node structure. */ mask = rdp->grpmask; -spin_lock (&rsp->ofl_lock); +raw_spin_lock (&rsp->ofl_lock);raw_spin_lock_irqsave_rcu_node (rnp, flags);/* Enforce GP memory-order guarantee. */ rdp->rcu_ofl_gp_seq =READ_ONCE (rsp->gp_seq); rdp->rcu_ofl_gp_flags =READ_ONCE (rsp->gp_flags);
@@ -3903,7 +3903,7 @@ static void rcu_cleanup_dying_idle_cpu(i } rnp->qsmaskinitnext &= ~mask;raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -spin_unlock (&rsp->ofl_lock); +raw_spin_unlock (&rsp->ofl_lock); }/*
kernel/rcu/tree.h +1 -1
@@ -368,7 +368,7 @@ struct rcu_state {char abbr;/* Abbreviated name. */ struct list_head flavors;/* List of RCU flavors. */ -spinlock_t ofl_lock____cacheline_internodealigned_in_smp ; +raw_spinlock_t ofl_lock____cacheline_internodealigned_in_smp ;/* Synchronize offline with */ /* GP pre-initialization. */ };
Quoted 8 lines · Mike Galbraith
1e64b15a4b10("rcu: Fix grace-period hangs due to race with CPU offline") addedspinlock_tofl_lockto thercu_statestructure, then takes it with preemption disabled during CPU offline, giving RT sleeping lock heartburn.Convert it to
raw_spinlock_t.Signed-off-by Mike Galbraith <[email protected]>
Good catch, applied and started testing.
I had to hand-apply this one due to the RCU flavor consolidation churn. Let's just say that your patch is the 160th in -rcu slated for the next merge window, putting us about 30% of the way to the Greg KH criterion. ;-) (That said, about 25 of those patches are LKMM rather than RCU.)
So please see below for my adaptation of your patch. Please let me know if I messed anything up.
Thanx, Paul
------------------------------------------------------------------------
commit f5bc811f274bb5ff6da5f84f6928d00711c28f09
Author: Mike Galbraith <[email protected]> Date: Wed Aug 15 09:05:29 2018 -0700
rcu: Convert rcu_state.ofl_lock to raw_spinlock_t
1e64b15a4b10 ("rcu: Fix grace-period hangs due to race with CPU offline") added spinlock_t ofl_lock to the rcu_state structure, then takes it with preemption disabled during CPU offline, which gives the -rt patchset's sleeping spinlock heartburn.
This commit therefore converts ->ofl_lock to raw_spinlock_t.
kernel/rcu/tree.c +6 -6
@@ -97,7 +97,7 @@ struct rcu_state rcu_state = { .abbr = RCU_ABBR, .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex), .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex), - .ofl_lock = __SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock), + .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock), };/* Dump rcu_node combining tree at boot to verify correct setup. */
@@ -1775,13 +1775,13 @@ static bool rcu_gp_init(void)*/ rcu_state.gp_state =RCU_GP_ONOFF ;rcu_for_each_leaf_node (rnp) { -spin_lock (&rcu_state.ofl_lock); +raw_spin_lock (&rcu_state.ofl_lock);raw_spin_lock_irq_rcu_node (rnp);if (rnp->qsmaskinit == rnp->qsmaskinitnext && !rnp->wait_blkd_tasks) {/* Nothing to do on this leaf rcu_node structure. */ raw_spin_unlock_irq_rcu_node (rnp); -spin_unlock (&rcu_state.ofl_lock); +raw_spin_unlock (&rcu_state.ofl_lock);continue ; }
@@ -1817,7 +1817,7 @@ static bool rcu_gp_init(void) }raw_spin_unlock_irq_rcu_node (rnp); -spin_unlock (&rcu_state.ofl_lock); +raw_spin_unlock (&rcu_state.ofl_lock); }rcu_gp_slow (gp_preinit_delay);/* Races with CPU hotplug. */
@@ -3376,7 +3376,7 @@ void rcu_report_dead(unsigned int cpu)/* Remove outgoing CPU from mask in the leaf rcu_node structure. */ mask = rdp->grpmask; -spin_lock (&rcu_state.ofl_lock); +raw_spin_lock (&rcu_state.ofl_lock);raw_spin_lock_irqsave_rcu_node (rnp, flags);/* Enforce GP memory-order guarantee. */ rdp->rcu_ofl_gp_seq =READ_ONCE (rcu_state.gp_seq); rdp->rcu_ofl_gp_flags =READ_ONCE (rcu_state.gp_flags);
@@ -3387,7 +3387,7 @@ void rcu_report_dead(unsigned int cpu) } rnp->qsmaskinitnext &= ~mask;raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -spin_unlock (&rcu_state.ofl_lock); +raw_spin_unlock (&rcu_state.ofl_lock);per_cpu (rcu_cpu_started, cpu) =0 ; }
kernel/rcu/tree.h +1 -1
@@ -343,7 +343,7 @@ struct rcu_state {const char *name;/* Name of structure. */ char abbr;/* Abbreviated name. */ -spinlock_t ofl_lock____cacheline_internodealigned_in_smp ; +raw_spinlock_t ofl_lock____cacheline_internodealigned_in_smp ;/* Synchronize offline with */ /* GP pre-initialization. */ };