rcu: more review feedback
cover rcu: more review feedback
This patchset applies the last of the review comments:
o Whitespace fixes, updated comments, and code movement.
o Improve an existing abstraction and introduce two additional
abstractions for traversing the rcu_node tree.
Issues remaining:
o I can force hangs given sufficient rcutorture testing in
conjunction with random CPU-hotplug operations. I am working on a simpler interface between RCU and CPU hotplug that I believe will clear this up.
o TREE_PREEMPT_RCU currently uses the TREE_RCU variant of
synchronize_rcu_expedited, which is wrong. I am working on an expedited variant for TREE_PREEMPT_RCU, but will fall back on synchronize_rcu() for the short term if it gives me much trouble.
o The IPI hyperactivity from call_rcu() noted by Nick Piggin.
I believe I have identified a clean design for a fix for this.
o Make RCU less dependent on IPIs for forcing grace periods
(but this might be best deferred to 2.6.33).
o RCU priority boosting (2.6.33!).
o TINY_PREEMPT_RCU (2.6.33!).
o Squeeze another 400 bytes or so out of TINY_RCU (2.6.33!).
PATCH 1/2 rcu: Apply review feedback from Josh Triplett, part 3
Whitespace fixes, updated comments, and trivial code movement.
o Fix whitespace error in RCU_HEAD_INIT()
o Move "So where is rcu_write_lock()" comment so that it does
not come between the rcu_read_unlock() header comment and the rcu_read_unlock() definition.
o Move the module_param statements for blimit, qhimark, and qlowmark
to immediately follow the corresponding definitions.
o In __rcu_offline_cpu(), move the assignment to rdp_me inside the
"if" statement, given that rdp_me is not used outside of that "if" statement.
--- include/linux/rcupdate.h | 14 +++++++------- kernel/rcutree.c | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-)
include/linux/rcupdate.h +7 -7
@@ -83,7 +83,7 @@ extern int rcu_scheduler_active;#error "Unknown RCU implementation specified to kernel configuration"#endif -#define RCU_HEAD_INIT { .next = NULL, .func = NULL } +#define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT#define INIT_RCU_HEAD (ptr) do { \ (ptr)->next = NULL; (ptr)->func = NULL; \
@@ -135,12 +135,6 @@ static inline void rcu_read_lock(void)rcu_read_acquire (); } -/** -* rcu_read_unlock - marks the end of an RCU read-side critical section. -* -* See rcu_read_lock() for more information. -*/ -/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not
@@ -150,6 +144,12 @@ static inline void rcu_read_lock(void)* used as well. RCU does not care how the writers keep out of each * others' way, as long as they do so. */ + +/** +* rcu_read_unlock - marks the end of an RCU read-side critical section. +* +* See rcu_read_lock() for more information. +*/ static inline void rcu_read_unlock (void ) {rcu_read_release ();
kernel/rcutree.c +5 -5
@@ -121,6 +121,10 @@ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark =10000 ;/* If this many pending, ignore blimit. */ static int qlowmark =100 ;/* Once only this many pending, use blimit. */ +module_param (blimit, int,0 ); +module_param (qhimark, int,0 ); +module_param (qlowmark, int,0 ); +static void force_quiescent_state (struct rcu_state *rsp,int relaxed);static int rcu_pending (int cpu);
@@ -873,8 +877,8 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)* indefinitely delay callbacks, you have far worse things to * be worrying about. */ - rdp_me = rsp->rda[smp_processor_id ()];if (rdp->nxtlist !=NULL ) { + rdp_me = rsp->rda[smp_processor_id ()]; *rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; rdp->nxtlist =NULL ;
@@ -1570,7 +1574,3 @@ void __init __rcu_init(void) }#include "rcutree_plugin.h" - -module_param (blimit, int,0 ); -module_param (qhimark, int,0 ); -module_param (qlowmark, int,0 );
Signature · 1 line
1.5.2.5
PATCH 2/2 rcu: Apply review feedback from Josh Triplett, part 4
In reply to Paul E. McKenneyFrom: Paul E. McKenney <[email protected]>
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones.
o Make RCU_INIT_FLAVOR() declare its own variables, removing
the need to declare them at each call site.
o Create an rcu_for_each_leaf() macro that scans the leaf nodes
of the rcu_node tree.
o Create an rcu_for_each_node_breadth_first() macro that does
a breadth-first traversal of the rcu_node tree, AKA stepping through the array in index-number order.
--- kernel/rcutree.c | 53 ++++++++++++++++++++++++---------------------- kernel/rcutree.h | 12 ++++++++++ kernel/rcutree_plugin.h | 4 --- 3 files changed, 40 insertions(+), 29 deletions(-)
kernel/rcutree.c +28 -25
@@ -461,8 +461,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp)long delta;unsigned long flags;struct rcu_node *rnp =rcu_get_root (rsp); -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ];/* Only let one CPU complain about others per time interval. */
@@ -473,18 +471,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);/* OK, time to rat on our buddy... */ printk (KERN_ERR "INFO: RCU detected CPU stalls:" ); -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) {rcu_print_task_stall (rnp); -if (rnp_cur->qsmask ==0 ) +if (rnp->qsmask ==0 )continue ; -for (cpu =0 ; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++) -if (rnp_cur->qsmask & (1UL << cpu)) -printk (" %d" , rnp_cur->grplo + cpu); +for (cpu =0 ; cpu <= rnp->grphi - rnp->grplo; cpu++) +if (rnp->qsmask & (1UL << cpu)) +printk (" %d" , rnp->grplo + cpu); }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start));
@@ -644,7 +648,7 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)* one corresponding to this CPU, due to the fact that we have * irqs disabled. */ -for (rnp = &rsp->node[0 ]; rnp < &rsp->node[NUM_RCU_NODES ]; rnp++) { +rcu_for_each_node_breadth_first (rsp, rnp) {spin_lock (&rnp->lock);/* irqs already disabled. */ rcu_preempt_check_blocked_tasks (rnp); rnp->qsmask = rnp->qsmaskinit;
@@ -1037,33 +1041,32 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,int cpu;unsigned long flags;unsigned long mask; -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ]; +struct rcu_node *rnp; -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) { mask =0 ; -spin_lock_irqsave (&rnp_cur->lock, flags); +spin_lock_irqsave (&rnp->lock, flags);if (rsp->completed != lastcomp) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; } -if (rnp_cur->qsmask ==0 ) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +if (rnp->qsmask ==0 ) { +spin_unlock_irqrestore (&rnp->lock, flags);continue ; } - cpu = rnp_cur->grplo; + cpu = rnp->grplo; bit =1 ; -for (; cpu <= rnp_cur->grphi; cpu++, bit <<=1 ) { -if ((rnp_cur->qsmask & bit) !=0 &&f (rsp->rda[cpu])) +for (; cpu <= rnp->grphi; cpu++, bit <<=1 ) { +if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; }if (mask !=0 && rsp->completed == lastcomp) { -/* cpu_quiet_msk() releases rnp_cur->lock. */ -cpu_quiet_msk (mask, rsp, rnp_cur, flags); +/* cpu_quiet_msk() releases rnp->lock. */ +cpu_quiet_msk (mask, rsp, rnp, flags);continue ; } -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags); }return 0 ; }
@@ -1545,6 +1548,10 @@ static void __init rcu_init_one(struct rcu_state *rsp)*/ #define RCU_INIT_FLAVOR (rsp, rcu_data) \ do { \ + int i; \ + int j; \ + struct rcu_node *rnp; \ + \ rcu_init_one(rsp); \ rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \ j = 0; \
@@ -1559,10 +1566,6 @@ do { \void __init __rcu_init (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -rcu_bootup_announce ();#ifdef CONFIG_RCU_CPU_STALL_DETECTOR printk (KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n" );
kernel/rcutree.h +12 -0
@@ -106,6 +106,18 @@ struct rcu_node {/* blocked_tasks[] array. */ }____cacheline_internodealigned_in_smp ; +/* +* Do a full breadth-first scan of the rcu_node structures for the +* specified rcu_state structure. +*/ +#define rcu_for_each_node_breadth_first (rsp, rnp) \ + for ((rnp) = &(rsp)->node[0]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) + +#define rcu_for_each_leaf_node (rsp, rnp) \ + for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) +/* Index values for nxttail array in struct rcu_data. */ #define RCU_DONE_TAIL 0/* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1/* Also RCU_NEXT_READY head. */
kernel/rcutree_plugin.h +0 -4
@@ -424,10 +424,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu)*/ static void __init __rcu_init_preempt (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -RCU_INIT_FLAVOR (&rcu_preempt_state, rcu_preempt_data); }
Signature · 1 line
1.5.2.5
Josh Triplett
maintainerRe: [PATCH tip/core/rcu 2/2] rcu: Apply review feedback from Josh Triplett, part 4
Quoted 7 lines · Paul E. McKenney
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones.
o Make
RCU_INIT_FLAVOR()declare its own variables, removing
the need to declare them at each call site.
Quoted 8 lines
o Create an
rcu_for_each_leaf()macro that scans the leaf nodes
of thercu_nodetree.o Create an
rcu_for_each_node_breadth_first()macro that does
a breadth-first traversal of thercu_nodetree, AKA stepping through the array in index-number order.Signed-off-by Paul E. McKenney <[email protected]>
I noticed one bit of unrelated code in this patch, which the commit message doesn't mention:
Quoted 2 lines
kernel/rcutree.c +0
-0
[...]
kernel/rcutree.c static void print_other_cpu_stall(struct rcu_state *rsp) · line 480
Quoted 11 lines
kernel/rcutree.c +6
-0@@ -473,18 +471,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);
- Josh Triplett
Re: [PATCH tip/core/rcu 1/2] rcu: Apply review feedback from Josh Triplett, part 3
In reply to Paul E. McKenneyQuoted 8 lines · Paul E. McKenney
Whitespace fixes, updated comments, and trivial code movement.
o Fix whitespace error in
RCU_HEAD_INIT()o Move "So where is
rcu_write_lock()" comment so that it does
not come between thercu_read_unlock()header comment and thercu_read_unlock()definition.
Quoted 8 lines
o Move the
module_paramstatements for blimit, qhimark, and qlowmark
to immediately follow the corresponding definitions.o In
__rcu_offline_cpu(), move the assignment tordp_meinside the
"if" statement, given thatrdp_meis not used outside of that "if" statement.Signed-off-by Paul E. McKenney <[email protected]>
Quoted 80 lines
--- include/linux/rcupdate.h | 14 +++++++------- kernel/rcutree.c | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-)include/linux/rcupdate.h +7
-7@@ -83,7 +83,7 @@ extern int rcu_scheduler_active;#error "Unknown RCU implementation specified to kernel configuration"#endif -#define RCU_HEAD_INIT { .next = NULL, .func = NULL } +#define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT#define INIT_RCU_HEAD (ptr) do { \ (ptr)->next = NULL; (ptr)->func = NULL; \@@ -135,12 +135,6 @@ static inline void rcu_read_lock(void)rcu_read_acquire (); } -/** -* rcu_read_unlock - marks the end of an RCU read-side critical section. -* -* See rcu_read_lock() for more information. -*/ -/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not @@ -150,6 +144,12 @@ static inline void rcu_read_lock(void)* used as well. RCU does not care how the writers keep out of each * others' way, as long as they do so. */ + +/** +* rcu_read_unlock - marks the end of an RCU read-side critical section. +* +* See rcu_read_lock() for more information. +*/ static inline void rcu_read_unlock (void ) {rcu_read_release ();kernel/rcutree.c +5
-5@@ -121,6 +121,10 @@ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark =10000 ;/* If this many pending, ignore blimit. */ static int qlowmark =100 ;/* Once only this many pending, use blimit. */ +module_param (blimit, int,0 ); +module_param (qhimark, int,0 ); +module_param (qlowmark, int,0 ); +static void force_quiescent_state (struct rcu_state *rsp,int relaxed);static int rcu_pending (int cpu);@@ -873,8 +877,8 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)* indefinitely delay callbacks, you have far worse things to * be worrying about. */ - rdp_me = rsp->rda[smp_processor_id ()];if (rdp->nxtlist !=NULL ) { + rdp_me = rsp->rda[smp_processor_id ()]; *rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; rdp->nxtlist =NULL ;@@ -1570,7 +1574,3 @@ void __init __rcu_init(void) }#include "rcutree_plugin.h" - -module_param (blimit, int,0 ); -module_param (qhimark, int,0 ); -module_param (qlowmark, int,0 );Signature · 1 line
1.5.2.5
Re: [PATCH tip/core/rcu 2/2] rcu: Apply review feedback from Josh Triplett, part 4
In reply to Josh TriplettQuoted 11 lines · Josh Triplett
Quoted 10 lines · Paul E. McKenney
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones.
o Make
RCU_INIT_FLAVOR()declare its own variables, removing
the need to declare them at each call site.o Create an
rcu_for_each_leaf()macro that scans the leaf nodes
of thercu_nodetree.
Quoted 8 lines
Quoted 5 lines
o Create an
rcu_for_each_node_breadth_first()macro that does
a breadth-first traversal of thercu_nodetree, AKA stepping through the array in index-number order.Signed-off-by Paul E. McKenney <[email protected]>I noticed one bit of unrelated code in this patch, which the commit message doesn't mention:
Good point -- this was not from our code review, but rather from my mini-code-review introducing the relevant fixes.
Quoted 14 lines
Quoted 2 lines
kernel/rcutree.c +0
-0[...]
kernel/rcutree.c static void print_other_cpu_stall(struct rcu_state *rsp) · line 480
Quoted 11 lines
kernel/rcutree.c +6
-0@@ -473,18 +471,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);
The old code only checked for tasks blocked in RCU read-side critical sections that were queued in the leaves of the rcu_node tree. Of course, if all of the CPUs corresponding to a given rcu_node leaf have gone offline since the tasks where queued, then they are moved to the root rcu_node structure. So we need to check for stalled tasks in the root rcu_node structure as well in the leaf rcu_node structures.
Thanx, Paul
cover rcu: v2: more review feedback
In reply to Paul E. McKenneyThis patchset applies the last of the review comments:
o Whitespace fixes, updated comments, and code movement.
o Improve an existing abstraction and introduce two additional
abstractions for traversing the rcu_node tree.
Change since v1 (http://lkml.org/lkml/2009/9/27/17):
o Update second patch's changelog
Issues remaining:
o I can force hangs given sufficient rcutorture testing in
conjunction with random CPU-hotplug operations. I am working on a simpler interface between RCU and CPU hotplug that I believe will clear this up.
o TREE_PREEMPT_RCU currently uses the TREE_RCU variant of
synchronize_rcu_expedited, which is wrong. I am working on an expedited variant for TREE_PREEMPT_RCU, but will fall back on synchronize_rcu() for the short term if it gives me much trouble.
o The IPI hyperactivity from call_rcu() noted by Nick Piggin.
I believe I have identified a clean design for a fix for this.
o Make RCU less dependent on IPIs for forcing grace periods
(but this might be best deferred to 2.6.33).
o RCU priority boosting (2.6.33!).
o TINY_PREEMPT_RCU (2.6.33!).
o Squeeze another 400 bytes or so out of TINY_RCU (2.6.33!).
b/include/linux/rcupdate.h | 14 +++++------ b/kernel/rcutree.c | 11 ++++----- b/kernel/rcutree.h | 12 ++++++++++ b/kernel/rcutree_plugin.h | 5 ---- kernel/rcutree.c | 53 +++++++++++++++++++++++---------------------- 5 files changed, 52 insertions(+), 43 deletions(-)
PATCH 2/2 rcu: Apply review feedback from Josh Triplett, part 4
From: Paul E. McKenney <[email protected]>
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones. It also fixes an RCU stall-warning bug found while making the other changes.
o Make RCU_INIT_FLAVOR() declare its own variables, removing
the need to declare them at each call site.
o Create an rcu_for_each_leaf() macro that scans the leaf nodes
of the rcu_node tree.
o Create an rcu_for_each_node_breadth_first() macro that does
a breadth-first traversal of the rcu_node tree, AKA stepping through the array in index-number order.
o If all CPUs corresponding to a given leaf rcu_node structure
go offline, then any tasks queued on that leaf will be moved to the root rcu_node structure. Therefore, the stall-warning code must dump out tasks queued on the root rcu_node structure as well as those queued on the leaf rcu_node structures.
--- kernel/rcutree.c | 53 ++++++++++++++++++++++++---------------------- kernel/rcutree.h | 12 ++++++++++ kernel/rcutree_plugin.h | 4 --- 3 files changed, 40 insertions(+), 29 deletions(-)
kernel/rcutree.c +28 -25
@@ -461,8 +461,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp)long delta;unsigned long flags;struct rcu_node *rnp =rcu_get_root (rsp); -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ];/* Only let one CPU complain about others per time interval. */
@@ -473,18 +471,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);/* OK, time to rat on our buddy... */ printk (KERN_ERR "INFO: RCU detected CPU stalls:" ); -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) {rcu_print_task_stall (rnp); -if (rnp_cur->qsmask ==0 ) +if (rnp->qsmask ==0 )continue ; -for (cpu =0 ; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++) -if (rnp_cur->qsmask & (1UL << cpu)) -printk (" %d" , rnp_cur->grplo + cpu); +for (cpu =0 ; cpu <= rnp->grphi - rnp->grplo; cpu++) +if (rnp->qsmask & (1UL << cpu)) +printk (" %d" , rnp->grplo + cpu); }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start));
@@ -644,7 +648,7 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)* one corresponding to this CPU, due to the fact that we have * irqs disabled. */ -for (rnp = &rsp->node[0 ]; rnp < &rsp->node[NUM_RCU_NODES ]; rnp++) { +rcu_for_each_node_breadth_first (rsp, rnp) {spin_lock (&rnp->lock);/* irqs already disabled. */ rcu_preempt_check_blocked_tasks (rnp); rnp->qsmask = rnp->qsmaskinit;
@@ -1037,33 +1041,32 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,int cpu;unsigned long flags;unsigned long mask; -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ]; +struct rcu_node *rnp; -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) { mask =0 ; -spin_lock_irqsave (&rnp_cur->lock, flags); +spin_lock_irqsave (&rnp->lock, flags);if (rsp->completed != lastcomp) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; } -if (rnp_cur->qsmask ==0 ) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +if (rnp->qsmask ==0 ) { +spin_unlock_irqrestore (&rnp->lock, flags);continue ; } - cpu = rnp_cur->grplo; + cpu = rnp->grplo; bit =1 ; -for (; cpu <= rnp_cur->grphi; cpu++, bit <<=1 ) { -if ((rnp_cur->qsmask & bit) !=0 &&f (rsp->rda[cpu])) +for (; cpu <= rnp->grphi; cpu++, bit <<=1 ) { +if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; }if (mask !=0 && rsp->completed == lastcomp) { -/* cpu_quiet_msk() releases rnp_cur->lock. */ -cpu_quiet_msk (mask, rsp, rnp_cur, flags); +/* cpu_quiet_msk() releases rnp->lock. */ +cpu_quiet_msk (mask, rsp, rnp, flags);continue ; } -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags); }return 0 ; }
@@ -1545,6 +1548,10 @@ static void __init rcu_init_one(struct rcu_state *rsp)*/ #define RCU_INIT_FLAVOR (rsp, rcu_data) \ do { \ + int i; \ + int j; \ + struct rcu_node *rnp; \ + \ rcu_init_one(rsp); \ rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \ j = 0; \
@@ -1559,10 +1566,6 @@ do { \void __init __rcu_init (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -rcu_bootup_announce ();#ifdef CONFIG_RCU_CPU_STALL_DETECTOR printk (KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n" );
kernel/rcutree.h +12 -0
@@ -106,6 +106,18 @@ struct rcu_node {/* blocked_tasks[] array. */ }____cacheline_internodealigned_in_smp ; +/* +* Do a full breadth-first scan of the rcu_node structures for the +* specified rcu_state structure. +*/ +#define rcu_for_each_node_breadth_first (rsp, rnp) \ + for ((rnp) = &(rsp)->node[0]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) + +#define rcu_for_each_leaf_node (rsp, rnp) \ + for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) +/* Index values for nxttail array in struct rcu_data. */ #define RCU_DONE_TAIL 0/* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1/* Also RCU_NEXT_READY head. */
kernel/rcutree_plugin.h +0 -4
@@ -424,10 +424,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu)*/ static void __init __rcu_init_preempt (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -RCU_INIT_FLAVOR (&rcu_preempt_state, rcu_preempt_data); }
Signature · 1 line
1.5.2.5
PATCH 1/2 rcu: Apply review feedback from Josh Triplett, part 3
In reply to Paul E. McKenneyWhitespace fixes, updated comments, and trivial code movement.
o Fix whitespace error in RCU_HEAD_INIT()
o Move "So where is rcu_write_lock()" comment so that it does
not come between the rcu_read_unlock() header comment and the rcu_read_unlock() definition.
o Move the module_param statements for blimit, qhimark, and qlowmark
to immediately follow the corresponding definitions.
o In __rcu_offline_cpu(), move the assignment to rdp_me inside the
"if" statement, given that rdp_me is not used outside of that "if" statement.
--- include/linux/rcupdate.h | 14 +++++++------- kernel/rcutree.c | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-)
include/linux/rcupdate.h +7 -7
@@ -83,7 +83,7 @@ extern int rcu_scheduler_active;#error "Unknown RCU implementation specified to kernel configuration"#endif -#define RCU_HEAD_INIT { .next = NULL, .func = NULL } +#define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT#define INIT_RCU_HEAD (ptr) do { \ (ptr)->next = NULL; (ptr)->func = NULL; \
@@ -135,12 +135,6 @@ static inline void rcu_read_lock(void)rcu_read_acquire (); } -/** -* rcu_read_unlock - marks the end of an RCU read-side critical section. -* -* See rcu_read_lock() for more information. -*/ -/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not
@@ -150,6 +144,12 @@ static inline void rcu_read_lock(void)* used as well. RCU does not care how the writers keep out of each * others' way, as long as they do so. */ + +/** +* rcu_read_unlock - marks the end of an RCU read-side critical section. +* +* See rcu_read_lock() for more information. +*/ static inline void rcu_read_unlock (void ) {rcu_read_release ();
kernel/rcutree.c +5 -5
@@ -121,6 +121,10 @@ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark =10000 ;/* If this many pending, ignore blimit. */ static int qlowmark =100 ;/* Once only this many pending, use blimit. */ +module_param (blimit, int,0 ); +module_param (qhimark, int,0 ); +module_param (qlowmark, int,0 ); +static void force_quiescent_state (struct rcu_state *rsp,int relaxed);static int rcu_pending (int cpu);
@@ -873,8 +877,8 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)* indefinitely delay callbacks, you have far worse things to * be worrying about. */ - rdp_me = rsp->rda[smp_processor_id ()];if (rdp->nxtlist !=NULL ) { + rdp_me = rsp->rda[smp_processor_id ()]; *rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; rdp->nxtlist =NULL ;
@@ -1570,7 +1574,3 @@ void __init __rcu_init(void) }#include "rcutree_plugin.h" - -module_param (blimit, int,0 ); -module_param (qhimark, int,0 ); -module_param (qlowmark, int,0 );
Signature · 1 line
1.5.2.5
Ingo Molnar
AppliedRe: [PATCH tip/core/rcu 0/2] rcu: v2: more review feedback
In reply to Paul E. McKenneyQuoted 10 lines · Paul E. McKenney
This patchset applies the last of the review comments:
o Whitespace fixes, updated comments, and code movement.
o Improve an existing abstraction and introduce two additional
abstractions for traversing thercu_nodetree.Change since v1 (http://lkml.org/lkml/2009/9/27/17):
o Update second patch's changelog
Applied, thanks a lot Paul!
Ingo
[tip:core/rcu] rcu: Clean up code based on review feedback from Josh Triplett, part 3
In reply to Paul E. McKenneyCommit-ID: 10905fde9671288030c616473eade4574f9894a1 Gitweb: http://git.kernel.org/tip/10905fde9671288030c616473eade4574f9894a1 Author: Paul E. McKenney <[email protected]> AuthorDate: Mon, 28 Sep 2009 07:46:32 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 28 Sep 2009 17:52:38 +0200
rcu: Clean up code based on review feedback from Josh Triplett, part 3
Whitespace fixes, updated comments, and trivial code movement.
o Fix whitespace error in RCU_HEAD_INIT()
o Move "So where is rcu_write_lock()" comment so that it does
not come between the rcu_read_unlock() header comment and the rcu_read_unlock() definition.
o Move the module_param statements for blimit, qhimark, and
qlowmark to immediately follow the corresponding definitions.
o In __rcu_offline_cpu(), move the assignment to rdp_me
inside the "if" statement, given that rdp_me is not used outside of that "if" statement.
LKML-Reference: <12541491931164-git-send-email->
--- include/linux/rcupdate.h | 14 +++++++------- kernel/rcutree.c | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-)
include/linux/rcupdate.h +7 -7
@@ -83,7 +83,7 @@ extern int rcu_scheduler_active;#error "Unknown RCU implementation specified to kernel configuration"#endif -#define RCU_HEAD_INIT { .next = NULL, .func = NULL } +#define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT#define INIT_RCU_HEAD (ptr) do { \ (ptr)->next = NULL; (ptr)->func = NULL; \
@@ -135,12 +135,6 @@ static inline void rcu_read_lock(void)rcu_read_acquire (); } -/** -* rcu_read_unlock - marks the end of an RCU read-side critical section. -* -* See rcu_read_lock() for more information. -*/ -/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not
@@ -150,6 +144,12 @@ static inline void rcu_read_lock(void)* used as well. RCU does not care how the writers keep out of each * others' way, as long as they do so. */ + +/** +* rcu_read_unlock - marks the end of an RCU read-side critical section. +* +* See rcu_read_lock() for more information. +*/ static inline void rcu_read_unlock (void ) {rcu_read_release ();
kernel/rcutree.c +5 -5
@@ -122,6 +122,10 @@ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark =10000 ;/* If this many pending, ignore blimit. */ static int qlowmark =100 ;/* Once only this many pending, use blimit. */ +module_param (blimit, int,0 ); +module_param (qhimark, int,0 ); +module_param (qlowmark, int,0 ); +static void force_quiescent_state (struct rcu_state *rsp,int relaxed);static int rcu_pending (int cpu);
@@ -878,8 +882,8 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)* indefinitely delay callbacks, you have far worse things to * be worrying about. */ - rdp_me = rsp->rda[smp_processor_id ()];if (rdp->nxtlist !=NULL ) { + rdp_me = rsp->rda[smp_processor_id ()]; *rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; rdp->nxtlist =NULL ;
@@ -1575,7 +1579,3 @@ void __init __rcu_init(void) }#include "rcutree_plugin.h" - -module_param (blimit, int,0 ); -module_param (qhimark, int,0 ); -module_param (qlowmark, int,0 );
[tip:core/rcu] rcu: Clean up code based on review feedback from Josh Triplett, part 4
In reply to Paul E. McKenneyCommit-ID: 3a60838d46a718a6bd0bd69cb04eea10d8d59ede Gitweb: http://git.kernel.org/tip/3a60838d46a718a6bd0bd69cb04eea10d8d59ede Author: Paul E. McKenney <[email protected]> AuthorDate: Mon, 28 Sep 2009 07:46:33 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 28 Sep 2009 17:52:39 +0200
rcu: Clean up code based on review feedback from Josh Triplett, part 4
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones. It also fixes an RCU stall-warning bug found while making the other changes.
o Make RCU_INIT_FLAVOR() declare its own variables, removing
the need to declare them at each call site.
o Create an rcu_for_each_leaf() macro that scans the leaf
nodes of the rcu_node tree.
o Create an rcu_for_each_node_breadth_first() macro that does
a breadth-first traversal of the rcu_node tree, AKA stepping through the array in index-number order.
o If all CPUs corresponding to a given leaf rcu_node
structure go offline, then any tasks queued on that leaf will be moved to the root rcu_node structure. Therefore, the stall-warning code must dump out tasks queued on the root rcu_node structure as well as those queued on the leaf rcu_node structures.
LKML-Reference: <12541491934126-git-send-email->
--- kernel/rcutree.c | 53 ++++++++++++++++++++++++---------------------- kernel/rcutree.h | 12 ++++++++++ kernel/rcutree_plugin.h | 4 --- 3 files changed, 40 insertions(+), 29 deletions(-)
kernel/rcutree.c +28 -25
@@ -462,8 +462,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp)long delta;unsigned long flags;struct rcu_node *rnp =rcu_get_root (rsp); -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ];/* Only let one CPU complain about others per time interval. */
@@ -474,18 +472,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);/* OK, time to rat on our buddy... */ printk (KERN_ERR "INFO: RCU detected CPU stalls:" ); -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) {rcu_print_task_stall (rnp); -if (rnp_cur->qsmask ==0 ) +if (rnp->qsmask ==0 )continue ; -for (cpu =0 ; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++) -if (rnp_cur->qsmask & (1UL << cpu)) -printk (" %d" , rnp_cur->grplo + cpu); +for (cpu =0 ; cpu <= rnp->grphi - rnp->grplo; cpu++) +if (rnp->qsmask & (1UL << cpu)) +printk (" %d" , rnp->grplo + cpu); }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start));
@@ -649,7 +653,7 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)* one corresponding to this CPU, due to the fact that we have * irqs disabled. */ -for (rnp = &rsp->node[0 ]; rnp < &rsp->node[NUM_RCU_NODES ]; rnp++) { +rcu_for_each_node_breadth_first (rsp, rnp) {spin_lock (&rnp->lock);/* irqs already disabled. */ rcu_preempt_check_blocked_tasks (rnp); rnp->qsmask = rnp->qsmaskinit;
@@ -1042,33 +1046,32 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,int cpu;unsigned long flags;unsigned long mask; -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ]; +struct rcu_node *rnp; -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) { mask =0 ; -spin_lock_irqsave (&rnp_cur->lock, flags); +spin_lock_irqsave (&rnp->lock, flags);if (rsp->completed != lastcomp) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; } -if (rnp_cur->qsmask ==0 ) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +if (rnp->qsmask ==0 ) { +spin_unlock_irqrestore (&rnp->lock, flags);continue ; } - cpu = rnp_cur->grplo; + cpu = rnp->grplo; bit =1 ; -for (; cpu <= rnp_cur->grphi; cpu++, bit <<=1 ) { -if ((rnp_cur->qsmask & bit) !=0 &&f (rsp->rda[cpu])) +for (; cpu <= rnp->grphi; cpu++, bit <<=1 ) { +if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; }if (mask !=0 && rsp->completed == lastcomp) { -/* cpu_quiet_msk() releases rnp_cur->lock. */ -cpu_quiet_msk (mask, rsp, rnp_cur, flags); +/* cpu_quiet_msk() releases rnp->lock. */ +cpu_quiet_msk (mask, rsp, rnp, flags);continue ; } -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags); }return 0 ; }
@@ -1550,6 +1553,10 @@ static void __init rcu_init_one(struct rcu_state *rsp)*/ #define RCU_INIT_FLAVOR (rsp, rcu_data) \ do { \ + int i; \ + int j; \ + struct rcu_node *rnp; \ + \ rcu_init_one(rsp); \ rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \ j = 0; \
@@ -1564,10 +1571,6 @@ do { \void __init __rcu_init (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -rcu_bootup_announce ();#ifdef CONFIG_RCU_CPU_STALL_DETECTOR printk (KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n" );
kernel/rcutree.h +12 -0
@@ -106,6 +106,18 @@ struct rcu_node {/* blocked_tasks[] array. */ }____cacheline_internodealigned_in_smp ; +/* +* Do a full breadth-first scan of the rcu_node structures for the +* specified rcu_state structure. +*/ +#define rcu_for_each_node_breadth_first (rsp, rnp) \ + for ((rnp) = &(rsp)->node[0]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) + +#define rcu_for_each_leaf_node (rsp, rnp) \ + for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) +/* Index values for nxttail array in struct rcu_data. */ #define RCU_DONE_TAIL 0/* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1/* Also RCU_NEXT_READY head. */
kernel/rcutree_plugin.h +0 -4
@@ -423,10 +423,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu)*/ static void __init __rcu_init_preempt (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -RCU_INIT_FLAVOR (&rcu_preempt_state, rcu_preempt_data); }
Re: [PATCH tip/core/rcu 0/2] rcu: v2: more review feedback
In reply to Paul E. McKenneyQuoted 37 lines · Paul E. McKenney
This patchset applies the last of the review comments:
o Whitespace fixes, updated comments, and code movement.
o Improve an existing abstraction and introduce two additional
abstractions for traversing thercu_nodetree.Change since v1 (http://lkml.org/lkml/2009/9/27/17):
o Update second patch's changelog
Issues remaining:
o I can force hangs given sufficient rcutorture testing in
conjunction with random CPU-hotplug operations. I am working on a simpler interface between RCU and CPU hotplug that I believe will clear this up.o
TREE_PREEMPT_RCUcurrently uses theTREE_RCUvariant of
synchronize_rcu_expedited, which is wrong. I am working on an expedited variant forTREE_PREEMPT_RCU, but will fall back onsynchronize_rcu()for the short term if it gives me much trouble.o The IPI hyperactivity from
call_rcu()noted by Nick Piggin.
I believe I have identified a clean design for a fix for this.o Make RCU less dependent on IPIs for forcing grace periods
(but this might be best deferred to 2.6.33).o RCU priority boosting (2.6.33!).
o
TINY_PREEMPT_RCU(2.6.33!).o Squeeze another 400 bytes or so out of
TINY_RCU(2.6.33!).
Quoted 6 lines
b/include/linux/rcupdate.h | 14 +++++------ b/kernel/rcutree.c | 11 ++++----- b/kernel/rcutree.h | 12 ++++++++++ b/kernel/rcutree_plugin.h | 5 ---- kernel/rcutree.c | 53 +++++++++++++++++++++++---------------------- 5 files changed, 52 insertions(+), 43 deletions(-)
For both patches:
[tip:core/rcu] rcu: Clean up code based on review feedback from Josh Triplett, part 3
In reply to Paul E. McKenneyCommit-ID: 3d76c082907e8f83c5d5c4572f38d53ad8f00c4b Gitweb: http://git.kernel.org/tip/3d76c082907e8f83c5d5c4572f38d53ad8f00c4b Author: Paul E. McKenney <[email protected]> AuthorDate: Mon, 28 Sep 2009 07:46:32 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Oct 2009 21:02:02 +0200
rcu: Clean up code based on review feedback from Josh Triplett, part 3
Whitespace fixes, updated comments, and trivial code movement.
o Fix whitespace error in RCU_HEAD_INIT()
o Move "So where is rcu_write_lock()" comment so that it does
not come between the rcu_read_unlock() header comment and the rcu_read_unlock() definition.
o Move the module_param statements for blimit, qhimark, and
qlowmark to immediately follow the corresponding definitions.
o In __rcu_offline_cpu(), move the assignment to rdp_me
inside the "if" statement, given that rdp_me is not used outside of that "if" statement.
LKML-Reference: <12541491931164-git-send-email->
--- include/linux/rcupdate.h | 14 +++++++------- kernel/rcutree.c | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-)
include/linux/rcupdate.h +7 -7
@@ -77,7 +77,7 @@ extern int rcu_scheduler_active;#error "Unknown RCU implementation specified to kernel configuration"#endif -#define RCU_HEAD_INIT { .next = NULL, .func = NULL } +#define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT#define INIT_RCU_HEAD (ptr) do { \ (ptr)->next = NULL; (ptr)->func = NULL; \
@@ -129,12 +129,6 @@ static inline void rcu_read_lock(void)rcu_read_acquire (); } -/** -* rcu_read_unlock - marks the end of an RCU read-side critical section. -* -* See rcu_read_lock() for more information. -*/ -/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not
@@ -144,6 +138,12 @@ static inline void rcu_read_lock(void)* used as well. RCU does not care how the writers keep out of each * others' way, as long as they do so. */ + +/** +* rcu_read_unlock - marks the end of an RCU read-side critical section. +* +* See rcu_read_lock() for more information. +*/ static inline void rcu_read_unlock (void ) {rcu_read_release ();
kernel/rcutree.c +5 -5
@@ -122,6 +122,10 @@ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark =10000 ;/* If this many pending, ignore blimit. */ static int qlowmark =100 ;/* Once only this many pending, use blimit. */ +module_param (blimit, int,0 ); +module_param (qhimark, int,0 ); +module_param (qlowmark, int,0 ); +static void force_quiescent_state (struct rcu_state *rsp,int relaxed);static int rcu_pending (int cpu);
@@ -878,8 +882,8 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)* indefinitely delay callbacks, you have far worse things to * be worrying about. */ - rdp_me = rsp->rda[smp_processor_id ()];if (rdp->nxtlist !=NULL ) { + rdp_me = rsp->rda[smp_processor_id ()]; *rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; rdp_me->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; rdp->nxtlist =NULL ;
@@ -1575,7 +1579,3 @@ void __init __rcu_init(void) }#include "rcutree_plugin.h" - -module_param (blimit, int,0 ); -module_param (qhimark, int,0 ); -module_param (qlowmark, int,0 );
[tip:core/rcu] rcu: Clean up code based on review feedback from Josh Triplett, part 4
In reply to Paul E. McKenneyCommit-ID: a0b6c9a78c41dc36732d6e1e90f0f2f57b29816f Gitweb: http://git.kernel.org/tip/a0b6c9a78c41dc36732d6e1e90f0f2f57b29816f Author: Paul E. McKenney <[email protected]> AuthorDate: Mon, 28 Sep 2009 07:46:33 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Oct 2009 21:02:04 +0200
rcu: Clean up code based on review feedback from Josh Triplett, part 4
These issues identified during an old-fashioned face-to-face code review extending over many hours. This group improves an existing abstraction and introduces two new ones. It also fixes an RCU stall-warning bug found while making the other changes.
o Make RCU_INIT_FLAVOR() declare its own variables, removing
the need to declare them at each call site.
o Create an rcu_for_each_leaf() macro that scans the leaf
nodes of the rcu_node tree.
o Create an rcu_for_each_node_breadth_first() macro that does
a breadth-first traversal of the rcu_node tree, AKA stepping through the array in index-number order.
o If all CPUs corresponding to a given leaf rcu_node
structure go offline, then any tasks queued on that leaf will be moved to the root rcu_node structure. Therefore, the stall-warning code must dump out tasks queued on the root rcu_node structure as well as those queued on the leaf rcu_node structures.
LKML-Reference: <12541491934126-git-send-email->
--- kernel/rcutree.c | 53 ++++++++++++++++++++++++---------------------- kernel/rcutree.h | 12 ++++++++++ kernel/rcutree_plugin.h | 4 --- 3 files changed, 40 insertions(+), 29 deletions(-)
kernel/rcutree.c +28 -25
@@ -462,8 +462,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp)long delta;unsigned long flags;struct rcu_node *rnp =rcu_get_root (rsp); -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ];/* Only let one CPU complain about others per time interval. */
@@ -474,18 +472,24 @@ static void print_other_cpu_stall(struct rcu_state *rsp)return ; } rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ; + +/* +* Now rat on any tasks that got kicked up to the root rcu_node +* due to CPU offlining. +*/ +rcu_print_task_stall (rnp);spin_unlock_irqrestore (&rnp->lock, flags);/* OK, time to rat on our buddy... */ printk (KERN_ERR "INFO: RCU detected CPU stalls:" ); -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) {rcu_print_task_stall (rnp); -if (rnp_cur->qsmask ==0 ) +if (rnp->qsmask ==0 )continue ; -for (cpu =0 ; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++) -if (rnp_cur->qsmask & (1UL << cpu)) -printk (" %d" , rnp_cur->grplo + cpu); +for (cpu =0 ; cpu <= rnp->grphi - rnp->grplo; cpu++) +if (rnp->qsmask & (1UL << cpu)) +printk (" %d" , rnp->grplo + cpu); }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start));
@@ -649,7 +653,7 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)* one corresponding to this CPU, due to the fact that we have * irqs disabled. */ -for (rnp = &rsp->node[0 ]; rnp < &rsp->node[NUM_RCU_NODES ]; rnp++) { +rcu_for_each_node_breadth_first (rsp, rnp) {spin_lock (&rnp->lock);/* irqs already disabled. */ rcu_preempt_check_blocked_tasks (rnp); rnp->qsmask = rnp->qsmaskinit;
@@ -1042,33 +1046,32 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,int cpu;unsigned long flags;unsigned long mask; -struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS -1 ]; -struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES ]; +struct rcu_node *rnp; -for (; rnp_cur < rnp_end; rnp_cur++) { +rcu_for_each_leaf_node (rsp, rnp) { mask =0 ; -spin_lock_irqsave (&rnp_cur->lock, flags); +spin_lock_irqsave (&rnp->lock, flags);if (rsp->completed != lastcomp) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags);return 1 ; } -if (rnp_cur->qsmask ==0 ) { -spin_unlock_irqrestore (&rnp_cur->lock, flags); +if (rnp->qsmask ==0 ) { +spin_unlock_irqrestore (&rnp->lock, flags);continue ; } - cpu = rnp_cur->grplo; + cpu = rnp->grplo; bit =1 ; -for (; cpu <= rnp_cur->grphi; cpu++, bit <<=1 ) { -if ((rnp_cur->qsmask & bit) !=0 &&f (rsp->rda[cpu])) +for (; cpu <= rnp->grphi; cpu++, bit <<=1 ) { +if ((rnp->qsmask & bit) !=0 &&f (rsp->rda[cpu])) mask |= bit; }if (mask !=0 && rsp->completed == lastcomp) { -/* cpu_quiet_msk() releases rnp_cur->lock. */ -cpu_quiet_msk (mask, rsp, rnp_cur, flags); +/* cpu_quiet_msk() releases rnp->lock. */ +cpu_quiet_msk (mask, rsp, rnp, flags);continue ; } -spin_unlock_irqrestore (&rnp_cur->lock, flags); +spin_unlock_irqrestore (&rnp->lock, flags); }return 0 ; }
@@ -1550,6 +1553,10 @@ static void __init rcu_init_one(struct rcu_state *rsp)*/ #define RCU_INIT_FLAVOR (rsp, rcu_data) \ do { \ + int i; \ + int j; \ + struct rcu_node *rnp; \ + \ rcu_init_one(rsp); \ rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \ j = 0; \
@@ -1564,10 +1571,6 @@ do { \void __init __rcu_init (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -rcu_bootup_announce ();#ifdef CONFIG_RCU_CPU_STALL_DETECTOR printk (KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n" );
kernel/rcutree.h +12 -0
@@ -106,6 +106,18 @@ struct rcu_node {/* blocked_tasks[] array. */ }____cacheline_internodealigned_in_smp ; +/* +* Do a full breadth-first scan of the rcu_node structures for the +* specified rcu_state structure. +*/ +#define rcu_for_each_node_breadth_first (rsp, rnp) \ + for ((rnp) = &(rsp)->node[0]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) + +#define rcu_for_each_leaf_node (rsp, rnp) \ + for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ + (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) +/* Index values for nxttail array in struct rcu_data. */ #define RCU_DONE_TAIL 0/* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1/* Also RCU_NEXT_READY head. */
kernel/rcutree_plugin.h +0 -4
@@ -423,10 +423,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu)*/ static void __init __rcu_init_preempt (void ) { -int i;/* All used by RCU_INIT_FLAVOR(). */ -int j; -struct rcu_node *rnp; -RCU_INIT_FLAVOR (&rcu_preempt_state, rcu_preempt_data); }