rcu: Enable rcu_normal_wake_from_gp on small systems
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH v3 1/2 rcu: Enable rcu_normal_wake_from_gp on small systems
Automatically enable the rcu_normal_wake_from_gp parameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.
This helps to reduce a latency of normal synchronize_rcu() API by waking up GP-waiters earlier and decoupling synchronize_rcu() callers from regular callback handling.
A benchmark running 64 parallel jobs(system with 64 CPUs) invoking synchronize_rcu() demonstrates a notable latency reduction with the setting enabled.
Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915
100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default>
<rcu_normal_wake_from_gp>
0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183
...
<rcu_normal_wake_from_gp>
--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)
kernel/rcu/tree.c +11 -3
@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;
@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }
@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq); +/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + } +/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v3 2/2 Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Update the documentation about rcu_normal_wake_from_gp parameter.
--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
Documentation/admin-guide/kernel-parameters.txt +2 -1
@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronous
Signature · 1 line
2.39.5
Uladzislau Rezki
maintainerRe: [PATCH v3 2/2] Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Quoted 25 lines · Uladzislau Rezki (Sony)
Update the documentation about
rcu_normal_wake_from_gpparameter.Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Documentation/admin-guide/kernel-parameters.txt +2
-1@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronousSignature · 1 line
2.39.5
Same this one. Are you planning?
--
Uladzislau Rezki
Re: [PATCH v3 2/2] Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Quoted 21 lines · Uladzislau Rezki
Quoted 20 lines · Uladzislau Rezki (Sony)
Update the documentation about
rcu_normal_wake_from_gpparameter.Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Documentation/admin-guide/kernel-parameters.txt +2
-1@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0.
Quoted 7 lines
Quoted 5 lines
Documentation/admin-guide/kernel-parameters.txt +0
-0rcuscale.gp_async= [KNL] Measure performance of asynchronousSignature · 1 line
2.39.5Same this one. Are you planning?
Yes, will be including these in the PR for next merge window.
- Neeraj
Quoted 2 lines
--
Uladzislau Rezki
Uladzislau Rezki
maintainerRe: [PATCH v3 2/2] Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Quoted 32 lines · Neeraj Upadhyay
Quoted 21 lines · Uladzislau Rezki
Quoted 20 lines · Uladzislau Rezki (Sony)
Update the documentation about
rcu_normal_wake_from_gpparameter.Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Documentation/admin-guide/kernel-parameters.txt +2
-1@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0.Quoted 7 lines
Quoted 5 lines
Documentation/admin-guide/kernel-parameters.txt +0
-0rcuscale.gp_async= [KNL] Measure performance of asynchronousSignature · 1 line
2.39.5Same this one. Are you planning?
Quoted 2 lines
Yes, will be including these in the PR for next merge window.
Appreciate it and thank you!
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 90 lines · Uladzislau Rezki (Sony)
Automatically enable the
rcu_normal_wake_from_gpparameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.This helps to reduce a latency of normal
synchronize_rcu()API by waking up GP-waiters earlier and decouplingsynchronize_rcu()callers from regular callback handling.A benchmark running 64 parallel jobs(system with 64 CPUs) invoking
synchronize_rcu()demonstrates a notable latency reduction with the setting enabled.Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default><
rcu_normal_wake_from_gp>0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183...
<rcu_normal_wake_from_gp>Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)kernel/rcu/tree.c +11
-3@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq); +/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + } +/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )Signature · 1 line
2.39.5
Neeraj, are you planning to take this for next merge window?
--
Uladzislau Rezlo
Quoted 79 lines · Uladzislau Rezki
Quoted 78 lines · Uladzislau Rezki (Sony)
Automatically enable the
rcu_normal_wake_from_gpparameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.This helps to reduce a latency of normal
synchronize_rcu()API by waking up GP-waiters earlier and decouplingsynchronize_rcu()callers from regular callback handling.A benchmark running 64 parallel jobs(system with 64 CPUs) invoking
synchronize_rcu()demonstrates a notable latency reduction with the setting enabled.Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default><
rcu_normal_wake_from_gp>0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183...
<rcu_normal_wake_from_gp>Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)kernel/rcu/tree.c +5
-3@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq);
Quoted 14 lines
Quoted 12 lines
kernel/rcu/tree.c +6
-0+/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + } +/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )Signature · 1 line
2.39.5Neeraj, are you planning to take this for next merge window?
Yes, I have queued v2 of these for testing here:
https://git.kernel.org/pub/scm/linux/kernel/git/neeraj.upadhyay/linux-rcu.git/log/?h=dev.01.07.2025
Will include them in PR for next merge window.
- Neeraj
Quoted 2 lines
--
Uladzislau Rezlo
Uladzislau Rezki
maintainerQuoted 97 lines · Neeraj Upadhyay
Quoted 79 lines · Uladzislau Rezki
Quoted 78 lines · Uladzislau Rezki (Sony)
Automatically enable the
rcu_normal_wake_from_gpparameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.This helps to reduce a latency of normal
synchronize_rcu()API by waking up GP-waiters earlier and decouplingsynchronize_rcu()callers from regular callback handling.A benchmark running 64 parallel jobs(system with 64 CPUs) invoking
synchronize_rcu()demonstrates a notable latency reduction with the setting enabled.Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default><
rcu_normal_wake_from_gp>0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183...
<rcu_normal_wake_from_gp>Reviewed-by Joel Fernandes <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)kernel/rcu/tree.c +5
-3@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq);Quoted 14 lines
Quoted 12 lines
kernel/rcu/tree.c +6
-0+/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + } +/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )Signature · 1 line
2.39.5Neeraj, are you planning to take this for next merge window?
Quoted 6 lines
Yes, I have queued v2 of these for testing here:
https://git.kernel.org/pub/scm/linux/kernel/git/neeraj.upadhyay/linux-rcu.git/log/?h=dev.01.07.2025
Will include them in PR for next merge window.
Thank you! I asked because i did not see both in our common RCU tree.
--
Uladzislau Rezki