rcu: join rcu_ctrlblk and rcu_state
PATCH 4/5 rcu: join rcu_ctrlblk and rcu_state
This patch moves rcu_state into the rcu_ctrlblk. I think there are no reasons why we should have 2 different variables to control rcu state. Every user of rcu_state has also "rcu_ctrlblk *rcp" in the parameter list.
include/linux/rcupdate.h renamed from include/linux/rcupdate.h~4_JOIN +4 -0
@@ -65,6 +65,10 @@ struct rcu_ctrlblk {long cur;/* Current batch number. */ long completed;/* Number of the last completed batch */ int next_pending;/* Is the next batch already waiting? */ + + spinlock_t lock ____cacheline_maxaligned_in_smp; +cpumask_t cpumask;/* CPUs that need to switch in order */ +/* for current batch to proceed. */ } ____cacheline_maxaligned_in_smp;/* Is batch a before batch b ? */
kernel/rcupdate.c renamed from kernel/rcupdate.c~4_JOIN +38 -44
185 lines not on this page · raw message
Quoted 4 lines · Oleg Nesterov
This patch moves
rcu_stateinto thercu_ctrlblk. I think there are no reasons why we should have 2 different variables to control rcu state. Every user ofrcu_statehas also "rcu_ctrlblk*rcp" in the parameter list.
This patch looks sane to me. It passes a short one-hour rcutorture on ppc64 and x86, firing up some overnight runs as well.
Dipankar, Manfred, any other concerns? Cacheline alignment? (Seems to me this code is far enough from the fastpath that this should not be a problem, but thought I should ask.)
Thanx, Paul
Quoted 214 lines
Signed-off-by Oleg Nesterov <[email protected]>include/linux/rcupdate.h renamed from include/linux/rcupdate.h~4_JOIN +4
-0@@ -65,6 +65,10 @@ struct rcu_ctrlblk {long cur;/* Current batch number. */ long completed;/* Number of the last completed batch */ int next_pending;/* Is the next batch already waiting? */ + + spinlock_t lock ____cacheline_maxaligned_in_smp; +cpumask_t cpumask;/* CPUs that need to switch in order */ +/* for current batch to proceed. */ } ____cacheline_maxaligned_in_smp;/* Is batch a before batch b ? */ kernel/rcupdate.c renamed from kernel/rcupdate.c~4_JOIN +38
-44@@ -49,22 +49,18 @@#include <linux/cpu.h> /* Definition for rcupdate control block. */ -struct rcu_ctrlblk rcu_ctrlblk = - { .cur =-300 , .completed =-300 }; -struct rcu_ctrlblk rcu_bh_ctrlblk = - { .cur =-300 , .completed =-300 }; - -/* Bookkeeping of the progress of the grace period */ -struct rcu_state { -spinlock_t lock;/* Guard this struct and writes to rcu_ctrlblk */ -cpumask_t cpumask;/* CPUs that need to switch in order */ -/* for current batch to proceed. */ +struct rcu_ctrlblk rcu_ctrlblk = { + .cur =-300 , + .completed =-300 , + .lock =SPIN_LOCK_UNLOCKED , + .cpumask =CPU_MASK_NONE , +}; +struct rcu_ctrlblk rcu_bh_ctrlblk = { + .cur =-300 , + .completed =-300 , + .lock =SPIN_LOCK_UNLOCKED , + .cpumask =CPU_MASK_NONE , }; - -static struct rcu_state rcu_state ____cacheline_maxaligned_in_smp = - {.lock =SPIN_LOCK_UNLOCKED , .cpumask =CPU_MASK_NONE }; -static struct rcu_state rcu_bh_state ____cacheline_maxaligned_in_smp = - {.lock =SPIN_LOCK_UNLOCKED , .cpumask =CPU_MASK_NONE }; DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L };@@ -233,13 +229,13 @@ static void rcu_do_batch(struct rcu_data* This is done by rcu_start_batch. The start is not broadcasted to * all cpus, they must pick this up by comparing rcp->cur with * rdp->quiescbatch. All cpus are recorded in the -* rcu_state.cpumask bitmap. +* rcu_ctrlblk.cpumask bitmap. * - All cpus must go through a quiescent state. * Since the start of the grace period is not broadcasted, at least two * calls to rcu_check_quiescent_state are required: * The first call just notices that a new grace period is running. The * following calls check if there was a quiescent state since the beginning -* of the grace period. If so, it updates rcu_state.cpumask. If +* of the grace period. If so, it updates rcu_ctrlblk.cpumask. If * the bitmap is empty, then the grace period is completed. * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace * period (if necessary). @@ -247,9 +243,9 @@ static void rcu_do_batch(struct rcu_data/* * Register a new batch of callbacks, and start it up if there is currently no * active batch and the batch to be registered has not already occurred. -* Caller must hold rcu_state.lock. +* Caller must hold rcu_ctrlblk.lock. */ -static void rcu_start_batch (struct rcu_ctrlblk *rcp,struct rcu_state *rsp) +static void rcu_start_batch (struct rcu_ctrlblk *rcp) {if (rcp->next_pending && rcp->completed == rcp->cur) {@@ -264,11 +260,11 @@ static void rcu_start_batch(struct rcu_c/* * Accessing nohz_cpu_mask before incrementing rcp->cur needs a * Barrier Otherwise it can cause tickless idle CPUs to be -* included in rsp->cpumask, which will extend graceperiods +* included in rcp->cpumask, which will extend graceperiods * unnecessarily. */ smp_mb (); -cpus_andnot (rsp->cpumask, cpu_online_map, nohz_cpu_mask); +cpus_andnot (rcp->cpumask, cpu_online_map, nohz_cpu_mask); } }@@ -278,13 +274,13 @@ static void rcu_start_batch(struct rcu_c* Clear it from the cpu mask and complete the grace period if it was the last * cpu. Start another grace period if someone has further entries pending */ -static void cpu_quiet (int cpu,struct rcu_ctrlblk *rcp,struct rcu_state *rsp) +static void cpu_quiet (int cpu,struct rcu_ctrlblk *rcp) { -cpu_clear (cpu, rsp->cpumask); -if (cpus_empty (rsp->cpumask)) { +cpu_clear (cpu, rcp->cpumask); +if (cpus_empty (rcp->cpumask)) {/* batch completed ! */ rcp->completed = rcp->cur; -rcu_start_batch (rcp, rsp); +rcu_start_batch (rcp); } }@@ -294,7 +290,7 @@ static void cpu_quiet(int cpu, struct rc* quiescent cycle, then indicate that it has done so. */ static void rcu_check_quiescent_state (struct rcu_ctrlblk *rcp, -struct rcu_state *rsp,struct rcu_data *rdp) +struct rcu_data *rdp) {if (rdp->quiescbatch != rcp->cur) {/* start new grace period: */ @@ -319,15 +315,15 @@ static void rcu_check_quiescent_state(streturn ; rdp->qs_pending =0 ; -spin_lock (&rsp->lock); +spin_lock (&rcp->lock);/* * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync * during cpu startup. Ignore the quiescent state. */ if (likely (rdp->quiescbatch == rcp->cur)) -cpu_quiet (rdp->cpu, rcp, rsp); +cpu_quiet (rdp->cpu, rcp); -spin_unlock (&rsp->lock); +spin_unlock (&rcp->lock); }@@ -348,16 +344,16 @@ static void rcu_move_batch(struct rcu_da }static void __rcu_offline_cpu (struct rcu_data *this_rdp, -struct rcu_ctrlblk *rcp,struct rcu_state *rsp,struct rcu_data *rdp) +struct rcu_ctrlblk *rcp,struct rcu_data *rdp) {/* if the cpu going offline owns the grace period * we can block indefinitely waiting for it, so flush * it here */ -spin_lock_bh (&rsp->lock); +spin_lock_bh (&rcp->lock);if (rcp->cur != rcp->completed) -cpu_quiet (rdp->cpu, rcp, rsp); -spin_unlock_bh (&rsp->lock); +cpu_quiet (rdp->cpu, rcp); +spin_unlock_bh (&rcp->lock);rcu_move_batch (this_rdp, rdp->curlist, rdp->curtail);rcu_move_batch (this_rdp, rdp->nxtlist, rdp->nxttail);@@ -367,9 +363,9 @@ static void rcu_offline_cpu(int cpu)struct rcu_data *this_rdp = &get_cpu_var (rcu_data);struct rcu_data *this_bh_rdp = &get_cpu_var (rcu_bh_data); -__rcu_offline_cpu (this_rdp, &rcu_ctrlblk, &rcu_state, +__rcu_offline_cpu (this_rdp, &rcu_ctrlblk, &per_cpu (rcu_data, cpu)); -__rcu_offline_cpu (this_bh_rdp, &rcu_bh_ctrlblk, &rcu_bh_state, +__rcu_offline_cpu (this_bh_rdp, &rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu));put_cpu_var (rcu_data);put_cpu_var (rcu_bh_data);@@ -388,7 +384,7 @@ static void rcu_offline_cpu(int cpu)* This does the RCU processing work from tasklet context. */ static void __rcu_process_callbacks (struct rcu_ctrlblk *rcp, -struct rcu_state *rsp,struct rcu_data *rdp) +struct rcu_data *rdp) {if (rdp->curlist && !rcu_batch_before (rcp->completed, rdp->batch)) { *rdp->donetail = rdp->curlist;@@ -418,25 +414,23 @@ static void __rcu_process_callbacks(struif (!rcp->next_pending) {/* and start it/schedule start if it's a new batch */ -spin_lock (&rsp->lock); +spin_lock (&rcp->lock); rcp->next_pending =1 ; -rcu_start_batch (rcp, rsp); -spin_unlock (&rsp->lock); +rcu_start_batch (rcp); +spin_unlock (&rcp->lock); } }else {local_irq_enable (); } -rcu_check_quiescent_state (rcp, rsp, rdp); +rcu_check_quiescent_state (rcp, rdp);if (rdp->donelist)rcu_do_batch (rdp); }static void rcu_process_callbacks (unsigned long unused) { -__rcu_process_callbacks (&rcu_ctrlblk, &rcu_state, - &__get_cpu_var (rcu_data)); -__rcu_process_callbacks (&rcu_bh_ctrlblk, &rcu_bh_state, - &__get_cpu_var (rcu_bh_data)); +__rcu_process_callbacks (&rcu_ctrlblk, &__get_cpu_var (rcu_data)); +__rcu_process_callbacks (&rcu_bh_ctrlblk, &__get_cpu_var (rcu_bh_data)); }static int __rcu_pending (struct rcu_ctrlblk *rcp,struct rcu_data *rdp)
Quoted 7 lines · Paul E. McKenney
This patch looks sane to me. It passes a short one-hour rcutorture on ppc64 and x86, firing up some overnight runs as well.
Dipankar, Manfred, any other concerns? Cacheline alignment? (Seems to me this code is far enough from the fastpath that this should not be a problem, but thought I should ask.)
I'd ask you and Oleg to re-synchronize, and perhaps Oleg to re-send the (part of?) the series that has no debate. I'm unsure, for example, whether #2 was just to be dropped.
I already applied #1, and it looks like there's agreement on #3 and #4, but basically, just to make sure, can Oleg please re-send to make sure I got it right?
Getting a screwed-up RCU thing is going to be too painful to debug, so I'd rather get it right the first time it hits my tree..
Linus
Quoted 6 lines · Linus Torvalds
Quoted 4 lines · Paul E. McKenney
This patch looks sane to me. It passes a short one-hour rcutorture on ppc64 and x86, firing up some overnight runs as well.
Quoted 7 lines
Quoted 3 lines
Dipankar, Manfred, any other concerns? Cacheline alignment? (Seems to me this code is far enough from the fastpath that this should not be a problem, but thought I should ask.)
I'd ask you and Oleg to re-synchronize, and perhaps Oleg to re-send the (part of?) the series that has no debate. I'm unsure, for example, whether #2 was just to be dropped.
I believe that the original #2 is to be dropped, but that the patch Oleg submitted in:
http://marc.theaimsgroup.com/?l=linux-kernel&m=113681388600342&w=2
may be needed. I have added Vatsa to the CC to get his take on this.
However, this patch should be independent from #4, so it should be OK to apply an updated #4 first while we are working out what to do about #2.
Quoted 6 lines
I already applied #1, and it looks like there's agreement on #3 and #4, but basically, just to make sure, can Oleg please re-send to make sure I got it right?
Getting a screwed-up RCU thing is going to be too painful to debug, so I'd rather get it right the first time it hits my tree..
Been there more times than I care to admit, and I most definitely agree!!!
Thanx, Paul
Quoted 6 lines · Paul E. McKenney
I believe that the original #2 is to be dropped, but that the patch Oleg submitted in:
http://marc.theaimsgroup.com/?l=linux-kernel&m=113681388600342&w=2may be needed. I have added Vatsa to the CC to get his take on this.
The patch submitted in the above URL seems fine to me and I think we should take it after Oleg does some basic testing.
- vatsa
Quoted 6 lines · Paul E. McKenney
Quoted 4 lines · Oleg Nesterov
This patch moves
rcu_stateinto thercu_ctrlblk. I think there are no reasons why we should have 2 different variables to control rcu state. Every user ofrcu_statehas also "rcu_ctrlblk*rcp" in the parameter list.
Quoted 7 lines
This patch looks sane to me. It passes a short one-hour rcutorture on ppc64 and x86, firing up some overnight runs as well.
Dipankar, Manfred, any other concerns? Cacheline alignment? (Seems to me this code is far enough from the fastpath that this should not be a problem, but thought I should ask.)
rcu_state came over from Manfred's RCU_HUGE patch IIRC. I don't think it is necessary to allocate rcu_state separately in the current mainline RCU code. So, the patch looks OK to me, but Manfred might see something that I am not seeing.
Thanks
Dipankar
[I haven't read the diff, just a short comment]
Quoted 7 lines · Dipankar Sarma
rcu_statecame over from Manfred'sRCU_HUGEpatch IIRC. I don't think it is necessary to allocatercu_stateseparately in the current mainline RCU code. So, the patch looks OK to me, but Manfred might see something that I am not seeing.
The two-level rcu code was never merged, I still plan to clean it up.
But the idea of splitting the control block and the state is used in the current code:
- __rcu_pending() is the hot path, it only performs a read access to rcu_ctrlblk.
- write accesses to the rcu_ctrlblk are really rare, they only happen when a new batch is started. Especially: independant from the number of cpus.
Write access to the rcu_state are common:
- each cpu must write once in each cycle to update it's cpu mask.
- The last cpu then completes the quiescent cycle.
The idea is that rcu_state is more or less write-only and rcu_state is read-only. Theoretically, rcu_state could be shared in all cpus caches, and there will be only one invalidate when a new batch is started. Thus no cacheline trashing due to rcu_pending calls. I think it would be safer to keep the two state counters in a separate cacheline from the spinlock and the cpu mask, but I don't have any hard numbers. IIRC the problems with the large SGI systems disappered, and everyone was happy. No real benchmark comparisons were made.
--
Manfred
Quoted 25 lines · Manfred Spraul
[I haven't read the diff, just a short comment]
Quoted 7 lines · Dipankar Sarma
rcu_statecame over from Manfred'sRCU_HUGEpatch IIRC. I don't think it is necessary to allocatercu_stateseparately in the current mainline RCU code. So, the patch looks OK to me, but Manfred might see something that I am not seeing.The two-level rcu code was never merged, I still plan to clean it up.
But the idea of splitting the control block and the state is used in the current code:
-__rcu_pending()is the hot path, it only performs a read access torcu_ctrlblk.
- write accesses to thercu_ctrlblkare really rare, they only happen when a new batch is started. Especially: independant from the number of cpus.Write access to the
rcu_stateare common:
- each cpu must write once in each cycle to update it's cpu mask.
- The last cpu then completes the quiescent cycle.
Quoted 8 lines
The idea is that
rcu_stateis more or less write-only andrcu_stateis read-only. Theoretically,rcu_statecould be shared in all cpus caches, and there will be only one invalidate when a new batch is started. Thus no cacheline trashing due torcu_pendingcalls.
I think it would be safer to keep the two state counters in a separate cacheline from the spinlock and the cpu mask, but I don't have any hard numbers. IIRC the problems with the large SGI systems disappered, and everyone was happy. No real benchmark comparisons were made.
Good point!
But doesn't the ____cacheline_maxaligned_in_smp directive on the "lock" field take care of this? Here is the structure:
/* Global control variables for rcupdate callback mechanism. */ struct rcu_ctrlblk {
long cur; /* Current batch number. */
long completed; /* Number of the last completed batch */
int next_pending; /* Is the next batch already waiting? */spinlock_t lock ____cacheline_internodealigned_in_smp; cpumask_t cpumask; /* CPUs that need to switch in order */ /* for current batch to proceed. */ } ____cacheline_internodealigned_in_smp;
If this does not cover this case, what more is needed?
Thanx, Paul