dynamic configuration for remote rcu callback processing
PATCH 2/2 dynamic configuration for remote rcu callback processing
This patch add the ability to dynamically configure processors for remote rcu callback processing. It applies on top of PATCH 1/2.
include/linux/rcupdate.h +1 -0
@@ -109,6 +109,7 @@ struct rcu_data {#ifdef CONFIG_SMP long last_rs_qlen;/* qlen during the last resched */ spinlock_t rmlock;/* for use with remote callback */ +short batch_stat;/* indicate processing being done */ #endif };
kernel/rcupdate.c +31 -8
@@ -195,7 +195,7 @@ static int rcu_next_remotercu(void)/* * Configure a cpu for remote rcu callback processing. */ -static int rcu_set_remote_rcu (int cpu) { +int rcu_set_remote_rcu (int cpu) {unsigned long flags;if (cpu <NR_CPUS ) {
@@ -206,11 +206,12 @@ static int rcu_set_remote_rcu(int cpu) { }else return 1 ; } +EXPORT_SYMBOL_GPL (rcu_set_remote_rcu);/* * Configure a cpu for standard rcu callback processing. */ -static void rcu_clear_remote_rcu (int cpu) { +void rcu_clear_remote_rcu (int cpu) {unsigned long flags;if (cpu <NR_CPUS ) {
@@ -219,6 +220,7 @@ static void rcu_clear_remote_rcu(int cpuspin_unlock_irqrestore (&cpu_remotercu_lock, flags); } } +EXPORT_SYMBOL_GPL (rcu_clear_remote_rcu);/* * Configure a set of cpus at boot time for remote rcu callback
@@ -238,7 +240,6 @@ static int __init rcu_remotercu_cpu_setu__setup ("remotercu=" , rcu_remotercu_cpu_setup);#else -static int rcu_callbacks_processed_remotely (int cpu) {return 0 ; }static int rcu_process_remote (int cpu) {return 0 ; }static void rcu_clear_remote_rcu (int cpu) {}#endif
@@ -650,8 +651,16 @@ static void __rcu_process_callbacks(stru }rcu_check_quiescent_state (rcp, rdp); -if (!rcu_callbacks_processed_remotely (cpu) && rdp->donelist) +#ifdef CONFIG_NUMA +if (rdp->donelist && !rcu_callbacks_processed_remotely (cpu) && + (cmpxchg (&rdp->batch_stat,0 ,1 )) ==0 ) {rcu_do_batch (rdp); + rdp->batch_stat =0 ; + } +#else +if (rdp->donelist) +rcu_do_batch (rdp); +#endif }static void rcu_process_callbacks (unsigned long unused)
@@ -692,15 +701,25 @@ static void rcu_process_remote_callbacks*/ rdp = &per_cpu (rcu_data, cpu);if (spin_trylock_irq (&rdp->rmlock)) { -if ((list=xchg (&rdp->donelist,NULL ))!=NULL ) - rdp->donetail = &rdp->donelist; +/* +* batch_stat ensures cpu isn't still running rcu_do_batch. +* This can happen if we've just configured on the fly. +*/ +if (cmpxchg (&rdp->batch_stat,0 ,2 ) ==0 ) { + list=xchg (&rdp->donelist,NULL ); +if (list !=NULL ) + rdp->donetail = &rdp->donelist; + }spin_unlock_irq (&rdp->rmlock); } rdp_bh = &per_cpu (rcu_bh_data, cpu);if (spin_trylock_irq (&rdp_bh->rmlock)) { -if ((list_bh=xchg (&rdp_bh->donelist,NULL ))!=NULL ) - rdp_bh->donetail = &rdp_bh->donelist; +if (cmpxchg (&rdp_bh->batch_stat,0 ,2 ) ==0 ) { + list_bh=xchg (&rdp_bh->donelist,NULL ); +if (list_bh !=NULL ) + rdp_bh->donetail = &rdp_bh->donelist; + }spin_unlock_irq (&rdp_bh->rmlock); }
@@ -717,6 +736,8 @@ static void rcu_process_remote_callbacks old = rdp->qlen; new = old - cnt; }while (cmpxchg (&rdp->qlen, old, new)!=old); +if (rdp->batch_stat ==2 ) + rdp->batch_stat =0 ; cnt=0 ;while (list_bh) {
@@ -729,6 +750,8 @@ static void rcu_process_remote_callbacks old = rdp_bh->qlen; new = old - cnt; }while (cmpxchg (&rdp_bh->qlen, old, new)!=old); +if (rdp_bh->batch_stat ==2 ) + rdp_bh->batch_stat =0 ; }#else static void rcu_process_remote_callbacks (unsigned long unused) {}
Quoted 2 lines · Dimitri Sivanich
This patch add the ability to dynamically configure processors for remote rcu callback processing. It applies on top of PATCH 1/2.
OK... So the reason some of the races in 1/2 were not a problem is that that patch did not allow any changes in the group of CPUs that have their RCU callbacks processed by other CPUs. However, I don't see how the patch below covers some of them. So I just called them out in the wrong patch. ;-)
Comments below.
Thanx, Paul
Quoted 59 lines
Signed-off-by Dimitri Sivanich <[email protected]>Signed-off-by Christoph Lameter <[email protected]>Index:
linux/include/linux/rcupdate.h
===================================================================include/linux/rcupdate.h +1
-0@@ -109,6 +109,7 @@ struct rcu_data {#ifdef CONFIG_SMP long last_rs_qlen;/* qlen during the last resched */ spinlock_t rmlock;/* for use with remote callback */ +short batch_stat;/* indicate processing being done */ #endif };Index:
linux/kernel/rcupdate.c
===================================================================kernel/rcupdate.c +4
-3@@ -195,7 +195,7 @@ static int rcu_next_remotercu(void)/* * Configure a cpu for remote rcu callback processing. */ -static int rcu_set_remote_rcu (int cpu) { +int rcu_set_remote_rcu (int cpu) {unsigned long flags;if (cpu <NR_CPUS ) {@@ -206,11 +206,12 @@ static int rcu_set_remote_rcu(int cpu) { }else return 1 ; } +EXPORT_SYMBOL_GPL (rcu_set_remote_rcu);/* * Configure a cpu for standard rcu callback processing. */ -static void rcu_clear_remote_rcu (int cpu) { +void rcu_clear_remote_rcu (int cpu) {unsigned long flags;if (cpu <NR_CPUS ) {@@ -219,6 +220,7 @@ static void rcu_clear_remote_rcu(int cpuspin_unlock_irqrestore (&cpu_remotercu_lock, flags); } } +EXPORT_SYMBOL_GPL (rcu_clear_remote_rcu);/* * Configure a set of cpus at boot time for remote rcu callback @@ -238,7 +240,6 @@ static int __init rcu_remotercu_cpu_setu__setup ("remotercu=" , rcu_remotercu_cpu_setup);#else -static int rcu_callbacks_processed_remotely (int cpu) {return 0 ; }static int rcu_process_remote (int cpu) {return 0 ; }static void rcu_clear_remote_rcu (int cpu) {}#endif
kernel/rcupdate.c static void __rcu_process_callbacks(stru · line 656
Quoted 7 lines
kernel/rcupdate.c +3
-1@@ -650,8 +651,16 @@ static void __rcu_process_callbacks(stru }rcu_check_quiescent_state (rcp, rdp); -if (!rcu_callbacks_processed_remotely (cpu) && rdp->donelist) +#ifdef CONFIG_NUMA +if (rdp->donelist && !rcu_callbacks_processed_remotely (cpu) && + (cmpxchg (&rdp->batch_stat,0 ,1 )) ==0 ) {
I have to ask... Why can't we just use a spinlock here? This code is effectively using batch_stat as a spinlock, right?
Quoted 11 lines
kernel/rcupdate.c +6
-0rcu_do_batch (rdp); + rdp->batch_stat = 0; + } +#else +if (rdp->donelist) +rcu_do_batch (rdp); +#endif }static void rcu_process_callbacks (unsigned long unused)
kernel/rcupdate.c static void rcu_process_remote_callbacks · line 708
Quoted 10 lines
kernel/rcupdate.c +5
-2@@ -692,15 +701,25 @@ static void rcu_process_remote_callbacks*/ rdp = &per_cpu (rcu_data, cpu);if (spin_trylock_irq (&rdp->rmlock)) { -if ((list=xchg (&rdp->donelist,NULL ))!=NULL ) - rdp->donetail = &rdp->donelist; +/* +* batch_stat ensures cpu isn't still running rcu_do_batch. +* This can happen if we've just configured on the fly. +*/ +if (cmpxchg (&rdp->batch_stat,0 ,2 ) ==0 ) {
Again, why not just a spinlock? The value 1 vs. 2 does not seem to be used.
Quoted 37 lines
kernel/rcupdate.c +13
-2+ list=xchg (&rdp->donelist,NULL ); +if (list !=NULL ) + rdp->donetail = &rdp->donelist; + }spin_unlock_irq (&rdp->rmlock); } rdp_bh = &per_cpu (rcu_bh_data, cpu);if (spin_trylock_irq (&rdp_bh->rmlock)) { -if ((list_bh=xchg (&rdp_bh->donelist,NULL ))!=NULL ) - rdp_bh->donetail = &rdp_bh->donelist; +if (cmpxchg (&rdp_bh->batch_stat,0 ,2 ) ==0 ) { + list_bh=xchg (&rdp_bh->donelist,NULL ); +if (list_bh !=NULL ) + rdp_bh->donetail = &rdp_bh->donelist; + }spin_unlock_irq (&rdp_bh->rmlock); }@@ -717,6 +736,8 @@ static void rcu_process_remote_callbacks old = rdp->qlen; new = old - cnt; }while (cmpxchg (&rdp->qlen, old, new)!=old); +if (rdp->batch_stat ==2 ) + rdp->batch_stat =0 ; cnt=0 ;while (list_bh) {@@ -729,6 +750,8 @@ static void rcu_process_remote_callbacks old = rdp_bh->qlen; new = old - cnt; }while (cmpxchg (&rdp_bh->qlen, old, new)!=old); +if (rdp_bh->batch_stat ==2 ) + rdp_bh->batch_stat =0 ; }#else static void rcu_process_remote_callbacks (unsigned long unused) {}