rcu: lastcomp cleanups, fix rcu_pending() false positives
cover rcu: lastcomp cleanups, fix rcu_pending() false positives
These two patches are yet more from the ongoing cleanup/inspection session for RCU.
1. Get the (hopefully) last of the accesses to the ->completed
fields cleaned up.
2. Fix a race in __rcu_pending() accesses to ->gpnum and
->completed fields that could cause needless (but otherwise harmless) raising of RCU_SOFTIRQ.
PATCH 1/2 rcu: further cleanups of use of lastcomp
Now that a copy of the rsp->completed flag is available in all rcu_node structures, make full use of it. It is still legitimate to access rsp->completed while holding the root rcu_node structure's lock, however. Also, tighten up force_quiescent_state()'s checks for end of current grace period.
--- kernel/rcutree.c | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-)
kernel/rcutree.c +6 -7
@@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) rnp = rdp->mynode;spin_lock_irqsave (&rnp->lock, flags); -if (lastcomp !=ACCESS_ONCE (rsp->completed)) { +if (lastcomp != rnp->completed) {/* * Someone beat us to it for this grace period, so leave.
@@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)static void __rcu_offline_cpu (int cpu,struct rcu_state *rsp) {unsigned long flags; -long lastcomp;unsigned long mask;struct rcu_data *rdp = rsp->rda[cpu];struct rcu_node *rnp;
@@ -958,7 +957,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)spin_unlock (&rnp->lock);/* irqs remain disabled. */ rnp = rnp->parent; }while (rnp !=NULL ); - lastcomp = rsp->completed;spin_unlock_irqrestore (&rsp->onofflock, flags);
@@ -1132,7 +1130,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,rcu_for_each_leaf_node (rsp, rnp) { mask =0 ;spin_lock_irqsave (&rnp->lock, flags); -if (rsp->completed != lastcomp) { +if (rnp->completed != lastcomp) {spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; }
@@ -1146,7 +1144,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; } -if (mask !=0 && rsp->completed == lastcomp) { +if (mask !=0 && rnp->completed == lastcomp) {/* cpu_quiet_msk() releases rnp->lock. */ cpu_quiet_msk (mask, rsp, rnp, flags);
@@ -1183,7 +1181,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) lastcomp = rsp->gpnum -1 ; signaled = rsp->signaled; rsp->jiffies_force_qs = jiffies +RCU_JIFFIES_TILL_FORCE_QS ; -if (lastcomp == rsp->gpnum) { +if (!rcu_gp_in_progress (rsp)) { rsp->n_force_qs_ngp++;spin_unlock (&rnp->lock); goto unlock_ret;/* no GP in progress, time updated. */
@@ -1211,7 +1209,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)/* Update state, record completion counter. */ forcenow =0 ;spin_lock (&rnp->lock); -if (lastcomp == rsp->completed && +if (lastcomp +1 == rsp->gpnum && + lastcomp == rsp->completed && rsp->signaled == signaled) { rsp->signaled =RCU_FORCE_QS ; rsp->completed_fqs = lastcomp;
Signature · 1 line
1.5.2.5
PATCH 2/2 rcu: eliminate __rcu_pending() false positives
In reply to Paul E. McKenneyFrom: Paul E. McKenney <[email protected]>
Now that there are both ->gpnum and ->completed fields in the rcu_node structure, __rcu_pending() should check rdp->gpnum and rdp->completed against rnp->gpnum and rdp->completed, respectively, instead of the prior comparison against the rcu_state fields rsp->gpnum and rsp->completed. Given the old comparison, __rcu_pending() could return 1, resulting in a needless raise_softirq(RCU_SOFTIRQ). This useless work would happen if RCU responded to a scheduling-clock interrupt after the rcu_state fields had been updated, but before the rcu_node fields had been updated.
Changing the comparison from the rcu_state fields to the rcu_node fields prevents this useless work from happening.
--- kernel/rcutree.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
kernel/rcutree.c +4 -2
@@ -1390,6 +1390,8 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);*/ static int __rcu_pending (struct rcu_state *rsp,struct rcu_data *rdp) { +struct rcu_node *rnp = rdp->mynode; + rdp->n_rcu_pending++;/* Check for CPU stalls, if enabled. */
@@ -1414,13 +1416,13 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) }/* Has another RCU grace period completed? */ -if (ACCESS_ONCE (rsp->completed) != rdp->completed) {/* outside lock */ +if (ACCESS_ONCE (rnp->completed) != rdp->completed) {/* outside lock */ rdp->n_rp_gp_completed++;return 1 ; }/* Has a new RCU grace period started? */ -if (ACCESS_ONCE (rsp->gpnum) != rdp->gpnum) {/* outside lock */ +if (ACCESS_ONCE (rnp->gpnum) != rdp->gpnum) {/* outside lock */ rdp->n_rp_gp_started++;return 1 ; }
Signature · 1 line
1.5.2.5
[tip:core/rcu] rcu: Further cleanups of use of lastcomp
In reply to Paul E. McKenneyCommit-ID: 560d4bc0df9a5e63b980432282d8c2bd3559ec74 Gitweb: http://git.kernel.org/tip/560d4bc0df9a5e63b980432282d8c2bd3559ec74 Author: Paul E. McKenney <[email protected]> AuthorDate: Fri, 13 Nov 2009 19:51:38 -0800 Committer: Ingo Molnar <[email protected]> CommitDate: Sat, 14 Nov 2009 10:31:42 +0100
rcu: Further cleanups of use of lastcomp
Now that a copy of the rsp->completed flag is available in all rcu_node structures, make full use of it. It is still legitimate to access rsp->completed while holding the root rcu_node structure's lock, however.
Also, tighten up force_quiescent_state()'s checks for end of current grace period.
LKML-Reference: <1258170699933-git-send-email->
--- kernel/rcutree.c | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-)
kernel/rcutree.c +6 -7
@@ -817,7 +817,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) rnp = rdp->mynode;spin_lock_irqsave (&rnp->lock, flags); -if (lastcomp !=ACCESS_ONCE (rsp->completed)) { +if (lastcomp != rnp->completed) {/* * Someone beat us to it for this grace period, so leave.
@@ -935,7 +935,6 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)static void __rcu_offline_cpu (int cpu,struct rcu_state *rsp) {unsigned long flags; -long lastcomp;unsigned long mask;struct rcu_data *rdp = rsp->rda[cpu];struct rcu_node *rnp;
@@ -971,7 +970,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)spin_unlock (&rnp->lock);/* irqs remain disabled. */ rnp = rnp->parent; }while (rnp !=NULL ); - lastcomp = rsp->completed;spin_unlock_irqrestore (&rsp->onofflock, flags);
@@ -1145,7 +1143,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,rcu_for_each_leaf_node (rsp, rnp) { mask =0 ;spin_lock_irqsave (&rnp->lock, flags); -if (rsp->completed != lastcomp) { +if (rnp->completed != lastcomp) {spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; }
@@ -1159,7 +1157,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; } -if (mask !=0 && rsp->completed == lastcomp) { +if (mask !=0 && rnp->completed == lastcomp) {/* cpu_quiet_msk() releases rnp->lock. */ cpu_quiet_msk (mask, rsp, rnp, flags);
@@ -1196,7 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) lastcomp = rsp->gpnum -1 ; signaled = rsp->signaled; rsp->jiffies_force_qs = jiffies +RCU_JIFFIES_TILL_FORCE_QS ; -if (lastcomp == rsp->gpnum) { +if (!rcu_gp_in_progress (rsp)) { rsp->n_force_qs_ngp++;spin_unlock (&rnp->lock); goto unlock_ret;/* no GP in progress, time updated. */
@@ -1224,7 +1222,8 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)/* Update state, record completion counter. */ forcenow =0 ;spin_lock (&rnp->lock); -if (lastcomp == rsp->completed && +if (lastcomp +1 == rsp->gpnum && + lastcomp == rsp->completed && rsp->signaled == signaled) { rsp->signaled =RCU_FORCE_QS ; rsp->completed_fqs = lastcomp;
[tip:core/rcu] rcu: Eliminate __rcu_pending() false positives
In reply to Paul E. McKenneyCommit-ID: 2f51f9884f6a36b0fe9636d5a1937e5cbd25723b Gitweb: http://git.kernel.org/tip/2f51f9884f6a36b0fe9636d5a1937e5cbd25723b Author: Paul E. McKenney <[email protected]> AuthorDate: Fri, 13 Nov 2009 19:51:39 -0800 Committer: Ingo Molnar <[email protected]> CommitDate: Sat, 14 Nov 2009 10:31:42 +0100
rcu: Eliminate __rcu_pending() false positives
Now that there are both ->gpnum and ->completed fields in the rcu_node structure, __rcu_pending() should check rdp->gpnum and rdp->completed against rnp->gpnum and rdp->completed, respectively, instead of the prior comparison against the rcu_state fields rsp->gpnum and rsp->completed.
Given the old comparison, __rcu_pending() could return 1, resulting in a needless raise_softirq(RCU_SOFTIRQ). This useless work would happen if RCU responded to a scheduling-clock interrupt after the rcu_state fields had been updated, but before the rcu_node fields had been updated.
Changing the comparison from the rcu_state fields to the rcu_node fields prevents this useless work from happening.
LKML-Reference: <12581706991966-git-send-email->
--- kernel/rcutree.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
kernel/rcutree.c +4 -2
@@ -1403,6 +1403,8 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);*/ static int __rcu_pending (struct rcu_state *rsp,struct rcu_data *rdp) { +struct rcu_node *rnp = rdp->mynode; + rdp->n_rcu_pending++;/* Check for CPU stalls, if enabled. */
@@ -1427,13 +1429,13 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) }/* Has another RCU grace period completed? */ -if (ACCESS_ONCE (rsp->completed) != rdp->completed) {/* outside lock */ +if (ACCESS_ONCE (rnp->completed) != rdp->completed) {/* outside lock */ rdp->n_rp_gp_completed++;return 1 ; }/* Has a new RCU grace period started? */ -if (ACCESS_ONCE (rsp->gpnum) != rdp->gpnum) {/* outside lock */ +if (ACCESS_ONCE (rnp->gpnum) != rdp->gpnum) {/* outside lock */ rdp->n_rp_gp_started++;return 1 ; }