rcu/kfree: Do not request RCU when not needed
RFC PATCH 1/1 rcu/kfree: Do not request RCU when not needed
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)
kernel/rcu/tree.c +17 -1
@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
@@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;
@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }
@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. +if (krcp->last_gp_seq && +rcu_seq_completed_gp (krcp->last_gp_seq, krcp->rdp->gp_seq)) { +queue_work (system_wq, &krwp->rcu_work.work); + }else { +queue_rcu_work (system_wq, &krwp->rcu_work); + } } }
@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp;
Signature · 1 line
2.38.1.273.g43a17bfeac-goog
Quoted 14 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +0
-0
kernel/rcu/tree.c struct kfree_rcu_cpu_work { · line 2940
Quoted 6 lines
kernel/rcu/tree.c +1
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
Yuck. I forgot to add last_gp_seq to the comments. Will await feedback before re-spin.
- Joel
Quoted 49 lines
kernel/rcu/tree.c +16
-2* @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. +if (krcp->last_gp_seq && +rcu_seq_completed_gp (krcp->last_gp_seq, krcp->rdp->gp_seq)) { +queue_work (system_wq, &krwp->rcu_work.work); + }else { +queue_rcu_work (system_wq, &krwp->rcu_work); + } } }@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; --2.38.1.273.g43a17bfeac-goog
Uladzislau Rezki
maintainerQuoted 39 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 10 lines
kernel/rcu/tree.c +6
-1// channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&
This check can be eliminated i think. A kfree_rcu_cpu is defined as static so by default the last_gp_set is set to zero.
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 8 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;
Yep. This one can be just dropped.
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
--
Uladzislau Rezki
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }
Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.
Ack.
Quoted 11 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample rdp->gp_seq, I think it is still worth a test.
Paul E. McKenney
maintainerQuoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.
Just for completeness, the main purpose of rdp->gp_seq is to reject quiescent states that were seen during already-completed grace periods.
Thanx, Paul
Uladzislau Rezki
maintainerQuoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.
So it means that instead of gp_seq reading we should take a snaphshot of the current state:
snp = get_state_synchronize_rcu();
and later on do a:
cond_synchronize_rcu(snp);
to wait for a GP. Or if the poll_state_synchronize_rcu(oldstate)) != 0 queue_rcu_work().
Sorry for a description using the RCU API functions name :)
--
Uladzislau Rezki
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:
Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().
But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
I believe you will have to call rcu_poll_gp_seq_start() as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.
(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
thanks,
- Joel
Quoted 2 lines
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 99 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
From 799ce1653d159ef3d35f34a284f738c2c267c75f Mon Sep 17 00:00:00 2001
From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu: kvfree_rcu: Reduce a memory footptint by using
polling APIs
Total time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh --memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfig CONFIG_NR_CPUS=64 \
--kconfig CONFIG_RCU_NOCB_CPU=y \
--kconfig CONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y \
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-make
According to data there is a big gain in memory footprint with a patch. It is because of call_rcu() and call_rcu_flush() take more effort and time to queue a callback and then wait for a gp.
With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.
--- kernel/rcu/tree.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)
kernel/rcu/tree.c +16 -5
@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/**
@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap); +raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {
@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {
@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }
@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; }
Signature · 4 lines
2.30.2 -- Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 181 lines · Uladzislau Rezki
Quoted 99 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
Quoted 1 line
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using
polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-)kernel/rcu/tree.c +4
-2@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/**
kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 9 lines
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);
Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
if (!poll_state_synchronize_rcu(krwp->gp_snap)) {
queue_rcu_work(system_wq, &krwp->rcu_work);
return;
}Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?
Thanx, Paul
Quoted 41 lines
kernel/rcu/tree.c +10
-2+raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; }Signature · 4 lines
2.30.2 -- Uladzislau Rezki
Quoted 200 lines · Paul E. McKenney
Quoted 181 lines · Uladzislau Rezki
Quoted 100 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>---
kernel/rcu/tree.c| 18 +++++++++++++++++-1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
Quoted 1 line
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>---
kernel/rcu/tree.c| 21 ++++++++++++++++-----1 file changed, 16 insertions(+), 5 deletions(-)kernel/rcu/tree.c +4
-2@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/** kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 9 lines
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?
The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the cond_synchronize_rcu() should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?
Thanks,
- Joel
Quoted 44 lines
Thanx, Paul
Quoted 41 lines
kernel/rcu/tree.c +10
-2+raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; }Signature · 4 lines
2.30.2 -- Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 209 lines · Joel Fernandes
Quoted 200 lines · Paul E. McKenney
Quoted 181 lines · Uladzislau Rezki
Quoted 100 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>---
kernel/rcu/tree.c| 18 +++++++++++++++++-1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
Quoted 1 line
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>---
kernel/rcu/tree.c| 21 ++++++++++++++++-----1 file changed, 16 insertions(+), 5 deletions(-)kernel/rcu/tree.c +4
-2@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/** kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 9 lines
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?
Quoted 5 lines
The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?
My concern is that the synchronize_rcu() will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.
In contrast, use of queue_rcu_work() frees up the kworker to handle other pages that are filling up.
Perhaps your point is that the delay from synchronize_rcu() should make the following pages take the fastpath through cond_synchronize_rcu()?
Either way, it might well be that context-switch overhead forces us to batch these things somehow. But let's worry about that when and if it actually happens.
Thanx, Paul
Quoted 48 lines
Thanks,
- Joel
Quoted 44 lines
Thanx, Paul
Quoted 41 lines
kernel/rcu/tree.c +10
-2+raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; }Signature · 4 lines
2.30.2 -- Uladzislau Rezki
Quoted 224 lines · Paul E. McKenney
Quoted 209 lines · Joel Fernandes
Quoted 200 lines · Paul E. McKenney
Quoted 181 lines · Uladzislau Rezki
Quoted 100 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>---
kernel/rcu/tree.c| 18 +++++++++++++++++-1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
Quoted 1 line
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>---
kernel/rcu/tree.c| 21 ++++++++++++++++-----1 file changed, 16 insertions(+), 5 deletions(-)kernel/rcu/tree.c +4
-2@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/** kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 9 lines
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?Quoted 5 lines
The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.In contrast, use of
queue_rcu_work()frees up the kworker to handle other pages that are filling up.
Quoted 6 lines
Perhaps your point is that the delay from
synchronize_rcu()should make the following pages take the fastpath throughcond_synchronize_rcu()?Either way, it might well be that context-switch overhead forces us to batch these things somehow. But let's worry about that when and if it actually happens.
Your point sounds reasonable. Though we'd hope cmwq scales worker thread count as needed, but we shouldn't probably stress it.
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the system_wq and block. (I might be throwing darts in the dark).
To be safe, we can implement your suggestion which is basically a form of my initial patch.
Should we add Tejun to the thread?
thanks,
- Joel
Quoted 51 lines
Thanx, Paul
Quoted 48 lines
Thanks,
- Joel
Quoted 44 lines
Thanx, Paul
Quoted 41 lines
kernel/rcu/tree.c +10
-3+raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } --2.30.2
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 228 lines · Joel Fernandes
Quoted 227 lines · Paul E. McKenney
Quoted 209 lines · Joel Fernandes
Quoted 200 lines · Paul E. McKenney
Quoted 181 lines · Uladzislau Rezki
Quoted 100 lines · Joel Fernandes
Quoted 81 lines · Uladzislau Rezki
Quoted 67 lines · Paul E. McKenney
Quoted 66 lines · Joel Fernandes
Quoted 39 lines · Uladzislau Rezki
Quoted 37 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>---
kernel/rcu/tree.c| 18 +++++++++++++++++-1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +4
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;queue_delayed_work (system_wq, &krcp->monitor_work, delay); }Quoted 14 lines
kernel/rcu/tree.c static void kfree_rcu_monitor(struct work_struct *work) · line 3230
Quoted 12 lines
kernel/rcu/tree.c +6
-1@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. + if (krcp->last_gp_seq &&This check can be eliminated i think. A
kfree_rcu_cpuis defined as static so by default thelast_gp_setis set to zero.Ack.
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4821
Quoted 7 lines
kernel/rcu/tree.c +2
-0@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;Yep. This one can be just dropped.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
But all the rest looks good :) I will give it a try from test point of view. It is interested from the memory footprint point of view.
Ack. Thanks. Even though we should not sample
rdp->gp_seq, I think it is still worth a test.Just for completeness, the main purpose of
rdp->gp_seqis to reject quiescent states that were seen during already-completed grace periods.So it means that instead of
gp_seqreading we should take a snaphshot of the current state:Quoted 7 lines
snp =
get_state_synchronize_rcu();and later on do a:
cond_synchronize_rcu(snp);to wait for a GP.
This can't be called from the timer IRQ handler though (monitor)
Quoted 2 lines
Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().But something like this should be possible (maybe)
Quoted 1 line
Sorry for a description using the RCU API functions name :)
Quoted 8 lines
I believe you will have to call
rcu_poll_gp_seq_start()as well if you are using polled API. I am planning to look at this properly more, soon. Right now I am going to write up the rcutop doc and share with you guys.(Maybe RCU polling is the right thing to do as we reuse all the infra and any corner case it is handling)
OK. This is in my todo list also. Since we have discussed it let's move it forward.
Below what i have came up with to switch for polling APIs:
Quoted 1 line
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches: 1110, memory footprint: 5057MB Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches: 1109, memory footprint: 2749MB Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches: 1158, memory footprint: 2934MB Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches: 981, memory footprint: 2704MB
with a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches: 1660, memory footprint: 91MB Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches: 1482, memory footprint: 86MB Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches: 1738, memory footprint: 86MB Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches: 1542, memory footprint: 72MB
Tested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>---
kernel/rcu/tree.c| 21 ++++++++++++++++-----1 file changed, 16 insertions(+), 5 deletions(-)kernel/rcu/tree.c +4
-2@@ -2919,18 +2919,20 @@ struct kvfree_rcu_bulk_data { ((PAGE_SIZE -sizeof (struct kvfree_rcu_bulk_data )) /sizeof (void *))/** +* @rcu_work: A work to reclaim a memory after a grace period * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests -* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period * @head_free: List of kfree_rcu() objects waiting for a grace period * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period * @krcp: Pointer to @kfree_rcu_cpu structure +* @gp_snap: A snapshot of current grace period */struct kfree_rcu_cpu_work { -struct rcu_work rcu_work; +struct work_struct rcu_work;struct rcu_head *head_free;struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS ];struct kfree_rcu_cpu *krcp; +unsigned long gp_snap; };/** kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 9 lines
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?Quoted 5 lines
The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.In contrast, use of
queue_rcu_work()frees up the kworker to handle other pages that are filling up.Perhaps your point is that the delay from
synchronize_rcu()should make the following pages take the fastpath throughcond_synchronize_rcu()?
Quoted 6 lines
Quoted 3 lines
Either way, it might well be that context-switch overhead forces us to batch these things somehow. But let's worry about that when and if it actually happens.
Your point sounds reasonable. Though we'd hope cmwq scales worker thread count as needed, but we shouldn't probably stress it.
It is especially inadvisable to stress it during OOM events, when the workqueue system might well get an allocation failure when it tries creating another kworker kthread. And we do want this to work well when at/near OOM, because this is one of the things that can free memory. ;-)
Quoted 10 lines
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the
system_wqand block. (I might be throwing darts in the dark).To be safe, we can implement your suggestion which is basically a form of my initial patch.
Should we add Tejun to the thread?
Let's get organized first, but that would be a good thing. Or I could reach out to Tejun internally.
For but one thing to get organized about, maybe kfree_rcu() should be using a workqueue with the WQ_MEM_RECLAIM flag set.
Thanx, Paul
Quoted 55 lines
thanks,
- Joel
Quoted 51 lines
Thanx, Paul
Quoted 48 lines
Thanks,
- Joel
Quoted 44 lines
Thanx, Paul
Quoted 41 lines
kernel/rcu/tree.c +10
-3+raw_spin_lock_irqsave (&krcp->lock, flags);// Channels 1 and 2. for (i =0 ; i <FREE_N_CHANNELS ; i++) {@@ -3194,6 +3198,13 @@ static void kfree_rcu_monitor(struct work_struct *work)if ((krcp->bkvhead[0 ] && !krwp->bkvhead_free[0 ]) || (krcp->bkvhead[1 ] && !krwp->bkvhead_free[1 ]) || (krcp->head && !krwp->head_free)) { +/* +* Take a snapshot for this krwp. Please note no +* more any objects can be added to this krwp free +* channels. +*/ + krwp->gp_snap =get_state_synchronize_rcu (); +// Channel 1 corresponds to the SLAB-pointer bulk path. // Channel 2 corresponds to vmalloc-pointer bulk path. for (j =0 ; j <FREE_N_CHANNELS ; j++) {@@ -3217,7 +3228,7 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +queue_work (system_wq, &krwp->rcu_work); } }@@ -4808,7 +4819,7 @@ static void __init kfree_rcu_batch_init(void)struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu);for (i =0 ; i <KFREE_N_BATCHES ; i++) { -INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); +INIT_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } --2.30.2
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 10 lines
Quoted 10 lines
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the
system_wqand block. (I might be throwing darts in the dark).To be safe, we can implement your suggestion which is basically a form of my initial patch.
Quoted 8 lines
Quoted 1 line
Should we add Tejun to the thread?
Let's get organized first, but that would be a good thing. Or I could reach out to Tejun internally.
For but one thing to get organized about, maybe
kfree_rcu()should be using a workqueue with theWQ_MEM_RECLAIMflag set.
It can be as an option to consider. Because such workqueue has some special priority for better handling of memory releasing. I can have a look at it closer to see how kvfree_rcu() works if it goes with WQ_MEM_RECLAIM.
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 15 lines · Uladzislau Rezki
Quoted 15 lines
Quoted 11 lines
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the
system_wqand block. (I might be throwing darts in the dark).To be safe, we can implement your suggestion which is basically a form of my initial patch.
Should we add Tejun to the thread?
Let's get organized first, but that would be a good thing. Or I could reach out to Tejun internally.
Quoted 7 lines
Quoted 3 lines
For but one thing to get organized about, maybe
kfree_rcu()should be using a workqueue with theWQ_MEM_RECLAIMflag set.It can be as an option to consider. Because such workqueue has some special priority for better handling of memory releasing. I can have a look at it closer to see how
kvfree_rcu()works if it goes withWQ_MEM_RECLAIM.
An extra note. It would work well with posted patch because we can directly queue the reclaim work to the WQ_MEM_RECLAIM queue.
As for now RCU-core kthreads like: rcugp, rcuop use "regular" queue. I think system_wq one.
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 23 lines · Uladzislau Rezki
Quoted 15 lines · Uladzislau Rezki
Quoted 15 lines
Quoted 11 lines
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the
system_wqand block. (I might be throwing darts in the dark).To be safe, we can implement your suggestion which is basically a form of my initial patch.
Should we add Tejun to the thread?
Let's get organized first, but that would be a good thing. Or I could reach out to Tejun internally.
Quoted 7 lines
Quoted 3 lines
For but one thing to get organized about, maybe
kfree_rcu()should be using a workqueue with theWQ_MEM_RECLAIMflag set.It can be as an option to consider. Because such workqueue has some special priority for better handling of memory releasing. I can have a look at it closer to see how
kvfree_rcu()works if it goes withWQ_MEM_RECLAIM.
Quoted 5 lines
An extra note. It would work well with posted patch because we can directly queue the reclaim work to the
WQ_MEM_RECLAIMqueue.As for now RCU-core kthreads like: rcugp, rcuop use "regular" queue. I think
system_wqone.
Agreed, it is the expedited grace period that uses WQ_MEM_RECLAIM workqueues.
Thanx, Paul
Paul E. McKenney
maintainerQuoted 15 lines · Uladzislau Rezki
Quoted 15 lines
Quoted 11 lines
Though I am thinking, workqueue context is normally used to invoke code that can block, and would the issue you mentioned affect those as well, or affect RCU when those non-RCU work items block. So for example, when other things in the system that can queue things on the
system_wqand block. (I might be throwing darts in the dark).To be safe, we can implement your suggestion which is basically a form of my initial patch.
Should we add Tejun to the thread?
Let's get organized first, but that would be a good thing. Or I could reach out to Tejun internally.
Quoted 6 lines
Quoted 3 lines
For but one thing to get organized about, maybe
kfree_rcu()should be using a workqueue with theWQ_MEM_RECLAIMflag set.It can be as an option to consider. Because such workqueue has some special priority for better handling of memory releasing. I can have a look at it closer to see how
kvfree_rcu()works if it goes withWQ_MEM_RECLAIM.
Sounds good, thank you!
Thanx, Paul
Uladzislau Rezki
maintainerQuoted 27 lines
Quoted 27 lines
Quoted 16 lines
kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 11 lines
/**
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Quoted 8 lines
Quoted 3 lines
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.
This concern works in both cases. I mean in default configuration and with a posted patch. The reclaim work, which name is kfree_rcu_work() only does a progress when a gp is passed so the rcu_work_rcufn() can queue our reclaim kworker.
As it is now:
1. Collect pointers, then we decide to drop them we queue the monitro_work() worker to the system_wq.
2. The monitor work, kfree_rcu_work(), tries to attach or saying it by another words bypass a "backlog" to "free" channels.
3. It invokes the queue_rcu_work() that does call_rcu_flush() and in its turn it queues our worker from the handler. So the worker is run after GP is passed.
With a patch:
[1] and [2] steps are the same. But on third step we do:
1. Record the GP status for last in channel;
2. Directly queue the drain work without any call_rcu() helpers;
3. On the reclaim worker entry we check if GP is passed;
4. If not it invokes synchronize_rcu().
The patch eliminates extra steps by not going via RCU-core route instead it directly invokes the reclaim worker where it either proceed or wait a GP if needed.
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 45 lines · Uladzislau Rezki
Quoted 27 lines
Quoted 27 lines
Quoted 16 lines
kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 11 lines
/**
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Quoted 8 lines
Quoted 3 lines
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.This concern works in both cases. I mean in default configuration and with a posted patch. The reclaim work, which name is
kfree_rcu_work()only does a progress when a gp is passed so thercu_work_rcufn()can queue our reclaim kworker.As it is now:
1. Collect pointers, then we decide to drop them we queue the
monitro_work()worker to thesystem_wq.
Quoted 6 lines
2. The monitor work,
kfree_rcu_work(), tries to attach or saying it by another words bypass a "backlog" to "free" channels.3. It invokes the
queue_rcu_work()that doescall_rcu_flush()and in its turn it queues our worker from the handler. So the worker is run after GP is passed.
So as it is now, we are not tying up a kworker kthread while waiting for the grace period, correct? We instead have an RCU callback queued during that time, and the kworker kthread gets involved only after the grace period ends.
Quoted 8 lines
With a patch:
[1] and [2] steps are the same. But on third step we do:
1. Record the GP status for last in channel;
2. Directly queue the drain work without anycall_rcu()helpers;
3. On the reclaim worker entry we check if GP is passed;
4. If not it invokessynchronize_rcu().
And #4 changes that, by (sometimes) tying up a kworker kthread for the full grace period.
Quoted 3 lines
The patch eliminates extra steps by not going via RCU-core route instead it directly invokes the reclaim worker where it either proceed or wait a GP if needed.
I agree that the use of the polled API could be reducing delays, which is a good thing. Just being my usual greedy self and asking "Why not both?", that is use queue_rcu_work() instead of synchronize_rcu() in conjunction with the polled APIs so as to avoid both the grace-period delay and the tying up of the kworker kthread.
Or am I missing something here?
Thanx, Paul
Quoted 78 lines · Paul E. McKenney
Quoted 45 lines · Uladzislau Rezki
Quoted 27 lines
Quoted 27 lines
Quoted 16 lines
kernel/rcu/tree.c static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 11 lines
/**
kernel/rcu/tree.c +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work)struct kfree_rcu_cpu_work *krwp;int i, j; - krwp =container_of (to_rcu_work (work), + krwp =container_of (work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; +cond_synchronize_rcu (krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return; }Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Quoted 8 lines
Quoted 3 lines
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.This concern works in both cases. I mean in default configuration and with a posted patch. The reclaim work, which name is
kfree_rcu_work()only does a progress when a gp is passed so thercu_work_rcufn()can queue our reclaim kworker.As it is now:
1. Collect pointers, then we decide to drop them we queue the
monitro_work()worker to thesystem_wq.Quoted 6 lines
2. The monitor work,
kfree_rcu_work(), tries to attach or saying it by another words bypass a "backlog" to "free" channels.3. It invokes the
queue_rcu_work()that doescall_rcu_flush()and in its turn it queues our worker from the handler. So the worker is run after GP is passed.So as it is now, we are not tying up a kworker kthread while waiting for the grace period, correct? We instead have an RCU callback queued during that time, and the kworker kthread gets involved only after the grace period ends.
Quoted 8 lines
With a patch:
[1] and [2] steps are the same. But on third step we do:
1. Record the GP status for last in channel;
2. Directly queue the drain work without anycall_rcu()helpers;
3. On the reclaim worker entry we check if GP is passed;
4. If not it invokessynchronize_rcu().And #4 changes that, by (sometimes) tying up a kworker kthread for the full grace period.
Quoted 3 lines
The patch eliminates extra steps by not going via RCU-core route instead it directly invokes the reclaim worker where it either proceed or wait a GP if needed.
Quoted 7 lines
I agree that the use of the polled API could be reducing delays, which is a good thing. Just being my usual greedy self and asking "Why not both?", that is use
queue_rcu_work()instead ofsynchronize_rcu()in conjunction with the polled APIs so as to avoid both the grace-period delay and the tying up of the kworker kthread.Or am I missing something here?
Yeah I am with Paul on this, NAK on “blocking in kworker” instead of “checking for grace period + queuing either regular work or RCU work”. Note that blocking also adds a pointless and fully avoidable scheduler round trip.
- Joel
Quoted 2 lines
Thanx, Paul
Quoted 88 lines · Joel Fernandes
Quoted 82 lines · Paul E. McKenney
Quoted 45 lines · Uladzislau Rezki
Quoted 27 lines
Quoted 27 lines
Quoted 16 lines
Review static void kfree_rcu_work(struct work_struct *work) · line 3075
Quoted 11 lines
/**
(quoted diff) +2
-1@@ -3066,10 +3068,12 @@ static void kfree_rcu_work(struct work_struct *work) struct kfree_rcu_cpu_work *krwp; int i, j; - krwp = container_of(to_rcu_work(work), + krwp = container_of(work, struct kfree_rcu_cpu_work, rcu_work); krcp = krwp->krcp; + cond_synchronize_rcu(krwp->gp_snap);Might this provoke OOMs in case of callback flooding?
An alternative might be something like this:
Quoted 8 lines
if (!poll_state_synchronize_rcu(krwp->gp_snap)) { queue_rcu_work(system_wq, &krwp->rcu_work); return;}
Either way gets you a non-lazy callback in the case where a grace period has not yet elapsed.
Or am I missing something that prevents OOMs here?The memory consumptions appears to be much less in his testing with the onslaught of kfree, which makes OOM probably less likely.
Quoted 8 lines
Quoted 3 lines
Though, was your reasoning that in case of a grace period not elapsing, we need a non lazy callback queued, so as to make the reclaim happen sooner?
If so, the
cond_synchronize_rcu()should already be conditionally queueing non-lazy CB since we don’t make synchronous users wait for seconds. Or did I miss something?My concern is that the
synchronize_rcu()will block a kworker kthread for some time, and that in callback-flood situations this might slow things down due to exhausting the supply of kworkers.This concern works in both cases. I mean in default configuration and with a posted patch. The reclaim work, which name is
kfree_rcu_work()only does a progress when a gp is passed so thercu_work_rcufn()can queue our reclaim kworker.As it is now:
1. Collect pointers, then we decide to drop them we queue the
monitro_work()worker to thesystem_wq.Quoted 6 lines
2. The monitor work,
kfree_rcu_work(), tries to attach or saying it by another words bypass a "backlog" to "free" channels.3. It invokes the
queue_rcu_work()that doescall_rcu_flush()and in its turn it queues our worker from the handler. So the worker is run after GP is passed.So as it is now, we are not tying up a kworker kthread while waiting for the grace period, correct? We instead have an RCU callback queued during that time, and the kworker kthread gets involved only after the grace period ends.
Quoted 8 lines
With a patch:
[1] and [2] steps are the same. But on third step we do:
1. Record the GP status for last in channel;
2. Directly queue the drain work without anycall_rcu()helpers;
3. On the reclaim worker entry we check if GP is passed;
4. If not it invokessynchronize_rcu().And #4 changes that, by (sometimes) tying up a kworker kthread for the full grace period.
Quoted 3 lines
The patch eliminates extra steps by not going via RCU-core route instead it directly invokes the reclaim worker where it either proceed or wait a GP if needed.
I agree that the use of the polled API could be reducing delays, which is a good thing. Just being my usual greedy self and asking "Why not both?", that is use
queue_rcu_work()instead ofsynchronize_rcu()in conjunction with the polled APIs so as to avoid both the grace-period delay and the tying up of the kworker kthread.
Quoted 3 lines
Quoted 1 line
Or am I missing something here?
Yeah I am with Paul on this, NAK on “blocking in kworker” instead of “checking for grace period + queuing either regular work or RCU work”. Note that blocking also adds a pointless and fully avoidable scheduler round trip.
As a side note, it’s notable how nicely this work evolved over the years thanks to Vlad and all of y’all’s work. For instance, flooding pages with kfree pointers and grace period polling was not something even invented back when kfree_rcu was a simple wrapper. Now it soon will be actually freeing memory faster, by avoiding waiting on RCU when not needed! And of course this is all happening probably because we wanted RCU to be lazy in nocb is a nice side effect of that effort ;-)
- Joel
Quoted 6 lines
- Joel
Quoted 2 lines
Thanx, Paul
Uladzislau Rezki
maintainerQuoted 7 lines
Quoted 4 lines
Quoted 2 lines
Or am I missing something here?
Yeah I am with Paul on this, NAK on “blocking in kworker” instead of “checking for grace period + queuing either regular work or RCU work”. Note that blocking also adds a pointless and fully avoidable scheduler round trip.
As a side note, it’s notable how nicely this work evolved over the years thanks to Vlad and all of y’all’s work. For instance, flooding pages with kfree pointers and grace period polling was not something even invented back when
kfree_rcuwas a simple wrapper. Now it soon will be actually freeing memory faster, by avoiding waiting on RCU when not needed! And of course this is all happening probably because we wanted RCU to be lazy in nocb is a nice side effect of that effort ;-)
Yep. We want sometimes everything to be lazy :)
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 14 lines
Quoted 14 lines
Quoted 1 line
This concern works in both cases. I mean in default configuration and with a posted patch. The reclaim work, which name is
kfree_rcu_work()only does a progress when a gp is passed so thercu_work_rcufn()can queue our reclaim kworker.As it is now:
1. Collect pointers, then we decide to drop them we queue the
monitro_work()worker to thesystem_wq.2. The monitor work,
kfree_rcu_work(), tries to attach or saying it by another words bypass a "backlog" to "free" channels.
Quoted 9 lines
Quoted 3 lines
3. It invokes the
queue_rcu_work()that doescall_rcu_flush()and in its turn it queues our worker from the handler. So the worker is run after GP is passed.So as it is now, we are not tying up a kworker kthread while waiting for the grace period, correct? We instead have an RCU callback queued during that time, and the kworker kthread gets involved only after the grace period ends.
Yes. The reclaim kworker is not kicked until a wrapper callback pushed via queue_rcu_work() -> call_rcu_flush() is invoked by the nocb-kthread:
<snip>
static void rcu_work_rcufn(struct rcu_head *rcu)
{
...
/* read the comment in __queue_work() */ local_irq_disable(); __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); local_irq_enable(); }
bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) {
...
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
rwork->wq = wq;
call_rcu_flush(&rwork->rcu, rcu_work_rcufn);..
<snip>
rcu_work_rcufn() callback runs our reclaim work after a full grace period. There is not so good dependency. If it is number, say, 10 000 in the cblist then we just wait for our time in a queue.
Quoted 4 lines
Quoted 4 lines
With a patch:
[1] and [2] steps are the same. But on third step we do:
Quoted 8 lines
Quoted 4 lines
1. Record the GP status for last in channel;
2. Directly queue the drain work without anycall_rcu()helpers;
3. On the reclaim worker entry we check if GP is passed;
4. If not it invokessynchronize_rcu().And #4 changes that, by (sometimes) tying up a kworker kthread for the full grace period.
Yes if GP is not passed we need to wait it anyway. Like this is done in default case. call_rcu()'s callback waits for a full grace period also and after that queues the work.
It is the same. But patched version can proceed reclaim right away if "during the fly" the GP is passed. On my tests i see it is ~30-40% of cases.
With a patch it also does not require go over RCU-core layer that is definitely longer.
Quoted 4 lines
Quoted 3 lines
The patch eliminates extra steps by not going via RCU-core route instead it directly invokes the reclaim worker where it either proceed or wait a GP if needed.
Quoted 8 lines
I agree that the use of the polled API could be reducing delays, which is a good thing. Just being my usual greedy self and asking "Why not both?", that is use
queue_rcu_work()instead ofsynchronize_rcu()in conjunction with the polled APIs so as to avoid both the grace-period delay and the tying up of the kworker kthread.Or am I missing something here?
Looks like not :)
1.
I was thinking about tracking a GP status for set of objects. With last in one we need to keep it updated for the whole set. Such objects can be freed faster, for example from monitor kworker. All other can go over queue_rcu_work().
This is more complex design but probably could be better. It will require split such chains for whom a GP is passed and not.
Please note there can be hidden problems and what i wrote might not be easily applicable.
The patch i sent is an attempt to optimize it by not introducing the big delta to current design.
2.call_rcu()
-> queue_work();
-> do reclaimqueue_work();
-> synchronize_rcu() or not if passed, do reclaimI can miss something though, why the latest is a bad way?
--
Uladzislau Rezki
A message that is not in the archive (sent off-list or not yet received): CAEXW_YQWYfJPpeXoV0ZDGC7Kd585LJ+h2YbKfB3unDDZinxTRQ@mail.gmail.com
Uladzislau Rezki
maintainerQuoted 56 lines
Nice, I am happy you worked on it. A comment bellow:
Below what i have came up with to switch for polling APIs:
Quoted 46 lines
From
799ce1653d159ef3d35f34a284f738c2c267c75fMon Sep 17 00:00:00 2001
From: "Uladzislau Rezki (Sony)" <[email protected]>
Date: Wed, 2 Nov 2022 19:26:27 +0100
Subject: [PATCH 1/1] rcu:kvfree_rcu: Reduce a memory footptint by using
polling APIsTotal time taken by all kfree'ers: 6564718459 ns, loops: 10000, batches:
1110, memory footprint: 5057MB
Total time taken by all kfree'ers: 8431051895 ns, loops: 10000, batches:
1109, memory footprint: 2749MB
Total time taken by all kfree'ers: 9477830789 ns, loops: 10000, batches:
1158, memory footprint: 2934MB
Total time taken by all kfree'ers: 9950211144 ns, loops: 10000, batches:
981, memory footprint: 2704MBwith a patch:
Total time taken by all kfree'ers: 7712110118 ns, loops: 10000, batches:
1660, memory footprint: 91MB
Total time taken by all kfree'ers: 7002403664 ns, loops: 10000, batches:
1482, memory footprint: 86MB
Total time taken by all kfree'ers: 7842282319 ns, loops: 10000, batches:
1738, memory footprint: 86MB
Total time taken by all kfree'ers: 7230161977 ns, loops: 10000, batches:
1542, memory footprint: 72MBTested with NOCB option, all offloading CPUs:
kvm.sh--memory 10G --torture rcuscale --allcpus --duration 1 \
--kconfigCONFIG_NR_CPUS=64\
--kconfigCONFIG_RCU_NOCB_CPU=y\
--kconfigCONFIG_RCU_NOCB_CPU_DEFAULT_ALL=y\
--bootargs "rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 \
rcuscale.holdoff=20 rcuscale.kfree_loops=10000
torture.disable_onoff_at_boot" --trust-makeAccording to data there is a big gain in memory footprint with a patch. It is because of
call_rcu()andcall_rcu_flush()take more effort and time to queue a callback and then wait for a gp.Quoted 4 lines
With polling API:
a) we do not need to queue any callback;
b) we might not even need wait for a GP completion.
Quoted 5 lines
Wow, I did not expect an extra grace period or 2 to cause so much memory unreclaimed.
Does this kernel have the 5 second timeout change?
Yes. But it does not matter. Why? Because 5 second time out happens only when a system lightly loaded. It can be 10 or 100 seconds.
Once a flood is detected we start to offload back-logged memory.
Quoted 5 lines
If yes, then I am confused why extra grace period matters for memory. My goal with the idea was to avoid pointless RCU work when we don’t need it. But I can’t complaint.
This patch does the same. It avoids of using any call_rcu() because we do not need it. I mean extra layer.
Why it matters for the memory. I explained it in previous answer. But it is because of extra layer. Since it is a flood scenario such delays can lead to higher memory footprint.
The test easily detects it.
Quoted 5 lines
Or is it that this test floods the pointer page so we do not wait full 5 seconds, and benefit when we reclaim sooner with polled API, than waiting for grace periods?
Right. Reclaim sooner with polling API.
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 4 lines
to wait for a GP. Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().
A small fix. If poll_state_synchronize_rcu(oldstate)) == 0 then queue_rcu_work() since a GP is still in progress.
Paul E. McKenney
maintainerQuoted 6 lines · Uladzislau Rezki
Quoted 4 lines
to wait for a GP. Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().A small fix. If
poll_state_synchronize_rcu(oldstate)) == 0 thenqueue_rcu_work()since a GP is still in progress.
Both get_state_synchronize_rcu() and poll_state_synchronize_rcu() may be invoked from interrupt handlers. And from NMI handlers, for that matter.
Thanx, Paul
Quoted 9 lines · Paul E. McKenney
Quoted 4 lines · Uladzislau Rezki
Quoted 4 lines
to wait for a GP. Or if the
poll_state_synchronize_rcu(oldstate)) != 0queue_rcu_work().
Quoted 6 lines
Quoted 2 lines
A small fix. If
poll_state_synchronize_rcu(oldstate)) == 0 thenqueue_rcu_work()since a GP is still in progress.Both
get_state_synchronize_rcu()andpoll_state_synchronize_rcu()may be invoked from interrupt handlers. And from NMI handlers, for that matter.
Thanks for clarifying. I meant to say cond_synchronize_rcu() cannot be called from interrupt context, but I made that comment before Vlad cooked his patch.
Thanks,
- Joel
Quoted 1 line
Thanx, Paul
Quoted 31 lines · Joel Fernandes (Google)
On ChromeOS, I am (almost) always seeing the optimization trigger. Tested boot up and trace_printk'ing how often it triggers.
Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)kernel/rcu/tree.c +3
-0@@ -2935,6 +2935,7 @@ struct kfree_rcu_cpu_work {/** * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period +* @rdp: The rdp of the CPU that this kfree_rcu corresponds to. * @head: List of kfree_rcu() objects not yet waiting for a grace period * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period @@ -2964,6 +2965,8 @@ struct kfree_rcu_cpu {struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES ];raw_spinlock_t lock;struct delayed_work monitor_work; +struct rcu_data *rdp; +unsigned long last_gp_seq;bool initialized;int count;
kernel/rcu/tree.c schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp) · line 3173
Quoted 4 lines
kernel/rcu/tree.c +1
-0@@ -3167,6 +3170,7 @@ schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)mod_delayed_work (system_wq, &krcp->monitor_work, delay);return ; } + krcp->last_gp_seq = krcp->rdp->gp_seq;
TODO: As Paul pointed out today in a meeting, we can't just sample rdp->gp_seq directly as it can jump forward quickly from out-of-date to non-out-of-date. Such GP can happen before a grace period actually elapses. Boom!
He did make me feel better though that it is something easy to miss, and to audit the current code about.
- Joel
- Joel
Quoted 34 lines
kernel/rcu/tree.c +13
-2queue_delayed_work (system_wq, &krcp->monitor_work, delay); }@@ -3217,7 +3221,17 @@ static void kfree_rcu_monitor(struct work_struct *work)// be that the work is in the pending state when // channels have been detached following by each // other. -queue_rcu_work (system_wq, &krwp->rcu_work); +// +// NOTE about gp_seq wrap: In case of gp_seq overflow, +// it is possible for rdp->gp_seq to be less than +// krcp->last_gp_seq even though a GP might be over. In +// this rare case, we would just have one extra GP. +if (krcp->last_gp_seq && +rcu_seq_completed_gp (krcp->last_gp_seq, krcp->rdp->gp_seq)) { +queue_work (system_wq, &krwp->rcu_work.work); + }else { +queue_rcu_work (system_wq, &krwp->rcu_work); + } } }@@ -4802,6 +4816,8 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + krcp->rdp =per_cpu_ptr (&rcu_data, cpu); + krcp->last_gp_seq =0 ;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; --2.38.1.273.g43a17bfeac-goog