rcu: Enable rcu_normal_wake_from_gp on small systems
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH v2 1/2 rcu: Enable rcu_normal_wake_from_gp on small systems
Automatically enable the rcu_normal_wake_from_gp parameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.
This helps to reduce a latency of normal synchronize_rcu() API by waking up GP-waiters earlier and decoupling synchronize_rcu() callers from regular callback handling.
A benchmark running 64 parallel jobs(system with 64 CPUs) invoking synchronize_rcu() demonstrates a notable latency reduction with the setting enabled.
Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915
100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default>
<rcu_normal_wake_from_gp>
0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183
...
<rcu_normal_wake_from_gp>
--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)
kernel/rcu/tree.c +11 -3
@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;
@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }
@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq); +/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + } +/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 2/2 Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Update the documentation about rcu_normal_wake_from_gp parameter.
--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
Documentation/admin-guide/kernel-parameters.txt +2 -1
@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronous
Signature · 1 line
2.39.5
Re: [PATCH v2 2/2] Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Quoted 3 lines · Uladzislau Rezki (Sony)
Update the documentation about
rcu_normal_wake_from_gpparameter.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>
Thanks.
Quoted 18 lines
--- Documentation/admin-guide/kernel-parameters.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Documentation/admin-guide/kernel-parameters.txt +2
-1@@ -5485,7 +5485,8 @@ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" - Default is 0. + Default is 1 if num_possible_cpus() <= 16 and it is not explicitly + disabled by the boot parameter passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronous
Uladzislau Rezki
maintainerRe: [PATCH v2 2/2] Documentation/kernel-parameters: Update rcu_normal_wake_from_gp doc
Quoted 9 lines · Joel Fernandes
Quoted 3 lines · Uladzislau Rezki (Sony)
Update the documentation about
rcu_normal_wake_from_gpparameter.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Reviewed-by Joel Fernandes <[email protected]>
Thanks!
--
Uladzislau Rezki
Quoted 11 lines · Uladzislau Rezki (Sony)
Automatically enable the
rcu_normal_wake_from_gpparameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.This helps to reduce a latency of normal
synchronize_rcu()API by waking up GP-waiters earlier and decouplingsynchronize_rcu()callers from regular callback handling.A benchmark running 64 parallel jobs(system with 64 CPUs) invoking
synchronize_rcu()demonstrates a notable latency reduction with the setting enabled.
Just one nit below:
Quoted 63 lines
Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default><
rcu_normal_wake_from_gp>0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183...
<rcu_normal_wake_from_gp>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)kernel/rcu/tree.c +5
-3@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }
kernel/rcu/tree.c void __init rcu_init(void) · line 4852
Quoted 8 lines
kernel/rcu/tree.c +5
-0@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq); +/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + }
nit: better to just do:
if (rcu_normal_wake_from_gp < 0 && num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) rcu_normal_wake_from_gp = 1; }
Quoted 4 lines
kernel/rcu/tree.c +1
-0+/* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld <0 )
Uladzislau Rezki
maintainerQuoted 11 lines · Joel Fernandes
Quoted 8 lines · Uladzislau Rezki (Sony)
Automatically enable the
rcu_normal_wake_from_gpparameter on systems with a small number of CPUs. The activation threshold is set to 16 CPUs.This helps to reduce a latency of normal
synchronize_rcu()API by waking up GP-waiters earlier and decouplingsynchronize_rcu()callers from regular callback handling.
Quoted 6 lines
Quoted 3 lines
A benchmark running 64 parallel jobs(system with 64 CPUs) invoking
synchronize_rcu()demonstrates a notable latency reduction with the setting enabled.Reviewed-by Joel Fernandes <[email protected]>
Thank you for review!
Quoted 76 lines
Just one nit below:
Quoted 63 lines
Latency distribution (microseconds):
<default>
0 - 9999 : 1 10000 - 19999 : 4 20000 - 29999 : 399 30000 - 39999 : 3197 40000 - 49999 : 10428 50000 - 59999 : 17363 60000 - 69999 : 15529 70000 - 79999 : 9287 80000 - 89999 : 4249 90000 - 99999 : 1915100000 - 109999 : 922
110000 - 119999 : 390
120000 - 129999 : 187
...
<default><
rcu_normal_wake_from_gp>0 - 9999 : 1 10000 - 19999 : 234 20000 - 29999 : 6678 30000 - 39999 : 33463 40000 - 49999 : 20669 50000 - 59999 : 2766 60000 - 69999 : 183...
<rcu_normal_wake_from_gp>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)kernel/rcu/tree.c +5
-3@@ -1625,8 +1625,10 @@ static void rcu_sr_put_wait_head(struct llist_node *node)atomic_set_release (&sr_wn->inuse,0 ); } -/* Disabled by default. */ -static int rcu_normal_wake_from_gp; +/* Enable rcu_normal_wake_from_gp automatically on small systems. */ +#define WAKE_FROM_GP_CPU_THRESHOLD 16 + +static int rcu_normal_wake_from_gp =-1 ;module_param (rcu_normal_wake_from_gp, int,0644 );static struct workqueue_struct *sync_wq;@@ -3239,7 +3241,7 @@ static void synchronize_rcu_normal(void)trace_rcu_sr_normal (rcu_state.name, &rs.head,TPS ("request" )); -if (!READ_ONCE (rcu_normal_wake_from_gp)) { +if (READ_ONCE (rcu_normal_wake_from_gp) <1 ) {wait_rcu_gp (call_rcu_hurry); goto trace_complete_out; }kernel/rcu/tree.c void __init rcu_init(void) · line 4852
Quoted 8 lines
kernel/rcu/tree.c +5
-0@@ -4843,6 +4845,12 @@ void __init rcu_init(void) sync_wq =alloc_workqueue ("sync_wq" ,WQ_MEM_RECLAIM ,0 );WARN_ON (!sync_wq); +/* Respect if explicitly disabled via a boot parameter. */ +if (rcu_normal_wake_from_gp <0 ) { +if (num_possible_cpus () <=WAKE_FROM_GP_CPU_THRESHOLD ) + rcu_normal_wake_from_gp =1 ; + }nit: better to just do:
Quoted 5 lines
if (rcu_normal_wake_from_gp < 0 && num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) rcu_normal_wake_from_gp = 1; }
Two lines anyway :)
--
Uladzislau Rezki