Move kvfree_rcu() into SLAB (v2)
Subsystems (3)
Uladzislau Rezki (Sony)
maintainerv2 cover Move kvfree_rcu() into SLAB (v2)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
https://lore.kernel.org/linux-mm/[email protected]/T/
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Uladzislau Rezki (Sony) (5):
rcu/kvfree: Initialize kvfree_rcu() separately rcu/kvfree: Move some functions under CONFIG_TINY_RCU rcu/kvfree: Adjust names passed into trace functions rcu/kvfree: Adjust a shrinker name
mm/slab: Move kvfree_rcu() into SLAB
include/linux/slab.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 876 ------------------------------------------ mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 882 insertions(+), 876 deletions(-)
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 1/5 rcu/kvfree: Initialize kvfree_rcu() separately
Introduce a separate initialization of kvfree_rcu() functionality. For such purpose a kfree_rcu_batch_init() is renamed to a kvfree_rcu_init() and it is invoked from the main.c right after rcu_init() is done.
--- include/linux/rcupdate.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-)
include/linux/rcupdate.h +1 -0
@@ -118,6 +118,7 @@ static inline void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func)/* Internal to kernel */ void rcu_init (void ); +void __init kvfree_rcu_init (void );extern int rcu_scheduler_active;void rcu_sched_clock_irq (int user);
init/main.c +1 -0
@@ -992,6 +992,7 @@ void start_kernel(void)workqueue_init_early ();rcu_init (); +kvfree_rcu_init ();/* Trace events are available after this */ trace_init ();
kernel/rcu/tree.c +1 -2
@@ -5648,7 +5648,7 @@ static void __init rcu_dump_rcu_node_tree(void)struct workqueue_struct *rcu_gp_wq; -static void __init kfree_rcu_batch_init (void ) +void __init kvfree_rcu_init (void ) {int cpu;int i, j;
@@ -5703,7 +5703,6 @@ void __init rcu_init(void)rcu_early_boot_tests (); -kfree_rcu_batch_init ();rcu_bootup_announce ();sanitize_kthread_prio ();rcu_init_geometry ();
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 2/5 rcu/kvfree: Move some functions under CONFIG_TINY_RCU
In reply to Uladzislau Rezki (Sony)Currently when a tiny RCU is enabled, the tree.c file is not compiled, thus duplicating function names do not conflict with each other.
Because of moving of kvfree_rcu() functionality to the SLAB, we have to reorder some functions and place them together under CONFIG_TINY_RCU macro definition. Therefore, those functions name will not conflict when a kernel is compiled for CONFIG_TINY_RCU flavor.
--- kernel/rcu/tree.c | 90 +++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +47 -43
@@ -3653,16 +3653,6 @@ static void kfree_rcu_monitor(struct work_struct *work)schedule_delayed_monitor_work (krcp); } -static enum hrtimer_restart -schedule_page_work_fn (struct hrtimer *t) -{ -struct kfree_rcu_cpu *krcp = - container_of(t, struct kfree_rcu_cpu, hrtimer); - -queue_delayed_work (system_highpri_wq, &krcp->page_cache_work,0 ); -return HRTIMER_NORESTART ; -} -static void fill_page_cache_func (struct work_struct *work) {struct kvfree_rcu_bulk_data *bnode;
@@ -3698,27 +3688,6 @@ static void fill_page_cache_func(struct work_struct *work)atomic_set (&krcp->backoff_page_cache_fill,0 ); } -static void -run_page_cache_worker (struct kfree_rcu_cpu *krcp) -{ -// If cache disabled, bail out. -if (!rcu_min_cached_objs) -return ; - -if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && - !atomic_xchg (&krcp->work_in_progress,1 )) { -if (atomic_read (&krcp->backoff_page_cache_fill)) { -queue_delayed_work (system_unbound_wq, - &krcp->page_cache_work, -msecs_to_jiffies (rcu_delay_page_cache_fill_msec)); - }else { -hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); - krcp->hrtimer.function = schedule_page_work_fn; -hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); - } - } -} -// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock() // state specified by flags. If can_alloc is true, the caller must // be schedulable and not be holding any locks or mutexes that might be
@@ -3779,6 +3748,51 @@ add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,return true; } +#if !defined(CONFIG_TINY_RCU ) + +static enum hrtimer_restart +schedule_page_work_fn (struct hrtimer *t) +{ +struct kfree_rcu_cpu *krcp = + container_of(t, struct kfree_rcu_cpu, hrtimer); + +queue_delayed_work (system_highpri_wq, &krcp->page_cache_work,0 ); +return HRTIMER_NORESTART ; +} + +static void +run_page_cache_worker (struct kfree_rcu_cpu *krcp) +{ +// If cache disabled, bail out. +if (!rcu_min_cached_objs) +return ; + +if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && + !atomic_xchg (&krcp->work_in_progress,1 )) { +if (atomic_read (&krcp->backoff_page_cache_fill)) { +queue_delayed_work (system_unbound_wq, + &krcp->page_cache_work, +msecs_to_jiffies (rcu_delay_page_cache_fill_msec)); + }else { +hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + krcp->hrtimer.function = schedule_page_work_fn; +hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); + } + } +} + +void __init kfree_rcu_scheduler_running (void ) +{ +int cpu; + +for_each_possible_cpu (cpu) { +struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); + +if (need_offload_krc (krcp)) +schedule_delayed_monitor_work (krcp); + } +} +/* * Queue a request for lazy invocation of the appropriate free routine * after a grace period. Please note that three paths are maintained,
@@ -3944,6 +3958,8 @@ void kvfree_rcu_barrier(void) }EXPORT_SYMBOL_GPL (kvfree_rcu_barrier); +#endif /* #if !defined(CONFIG_TINY_RCU) */ +static unsigned long kfree_rcu_shrink_count (struct shrinker *shrink,struct shrink_control *sc) {
@@ -3985,18 +4001,6 @@ kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)return freed ==0 ?SHRINK_STOP : freed; } -void __init kfree_rcu_scheduler_running (void ) -{ -int cpu; - -for_each_possible_cpu (cpu) { -struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); - -if (need_offload_krc (krcp)) -schedule_delayed_monitor_work (krcp); - } -} -/* * During early boot, any blocking grace-period wait automatically * implies a grace period.
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 3/5 rcu/kvfree: Adjust names passed into trace functions
In reply to Uladzislau Rezki (Sony)Currently trace functions are supplied with "rcu_state.name" member which is located in the structure. The problem is that the "rcu_state" structure variable is local and can not be accessed from another place.
To address this, this preparation patch passes "slab" string as a first argument.
--- kernel/rcu/tree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/rcu/tree.c +3 -3
@@ -3379,14 +3379,14 @@ kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp,rcu_lock_acquire (&rcu_callback_map);if (idx ==0 ) {// kmalloc() / kfree(). trace_rcu_invoke_kfree_bulk_callback ( - rcu_state.name, bnode->nr_records, +"slab" , bnode->nr_records, bnode->records);kfree_bulk (bnode->nr_records, bnode->records); }else {// vmalloc() / vfree(). for (i =0 ; i < bnode->nr_records; i++) {trace_rcu_invoke_kvfree_callback ( - rcu_state.name, bnode->records[i],0 ); +"slab" , bnode->records[i],0 );vfree (bnode->records[i]); }
@@ -3417,7 +3417,7 @@ kvfree_rcu_list(struct rcu_head *head) next = head->next;debug_rcu_head_unqueue ((struct rcu_head *)ptr);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_kvfree_callback (rcu_state.name, head, offset); +trace_rcu_invoke_kvfree_callback ("slab" , head, offset);if (!WARN_ON_ONCE (!__is_kvfree_rcu_offset (offset)))kvfree (ptr);
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 4/5 rcu/kvfree: Adjust a shrinker name
In reply to Uladzislau Rezki (Sony)Rename "rcu-kfree" to "slab-kvfree-rcu" since it goes to the slab_common.c file soon.
--- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/tree.c +1 -1
@@ -5689,7 +5689,7 @@ void __init kvfree_rcu_init(void) krcp->initialized = true; } - kfree_rcu_shrinker =shrinker_alloc (0 ,"rcu-kfree" ); + kfree_rcu_shrinker =shrinker_alloc (0 ,"slab-kvfree-rcu" );if (!kfree_rcu_shrinker) {pr_err ("Failed to allocate kfree_rcu() shrinker!\n" );return ;
Signature · 1 line
2.39.5
Uladzislau Rezki (Sony)
maintainerPATCH v2 5/5 mm/slab: Move kvfree_rcu() into SLAB
In reply to Uladzislau Rezki (Sony)Move kvfree_rcu() functionality to the slab_common.c file.
--- include/linux/rcupdate.h | 1 - include/linux/slab.h | 1 + kernel/rcu/tree.c | 879 -------------------------------------- mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 881 insertions(+), 880 deletions(-)
include/linux/rcupdate.h +0 -1
@@ -118,7 +118,6 @@ static inline void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func)/* Internal to kernel */ void rcu_init (void ); -void __init kvfree_rcu_init (void );extern int rcu_scheduler_active;void rcu_sched_clock_irq (int user);
include/linux/slab.h +1 -0
@@ -1099,5 +1099,6 @@ unsigned int kmem_cache_size(struct kmem_cache *s);size_t kmalloc_size_roundup (size_t size);void __init kmem_cache_init_late (void ); +void __init kvfree_rcu_init (void );#endif /* _LINUX_SLAB_H */
kernel/rcu/tree.c +0 -879
897 lines not on this page · raw message
mm/slab_common.c +880 -0
890 lines not on this page · raw message
Signature · 1 line
2.39.5
Christoph Lameter (Ampere)
reviewerQuoted 2 lines · Uladzislau Rezki (Sony)
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
I do not see any use of internal slab interfaces by this code. It seems to be using rcu internals though. So it would best be placed with the rcu code.
Uladzislau Rezki
maintainerQuoted 9 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Uladzislau Rezki (Sony)
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
I do not see any use of internal slab interfaces by this code. It seems to be using rcu internals though. So it would best be placed with the rcu code.
I think, later on there will be integration. This is a step forward to place it under mm where it should be.
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 8 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Uladzislau Rezki (Sony)
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
I do not see any use of internal slab interfaces by this code. It seems to be using rcu internals though. So it would best be placed with the rcu code.
That is indeed the current state. The point of moving it is to later take advantage of internal slab state.
Thanx, Paul
Uladzislau Rezki
maintainerQuoted 6 lines · Paul E. McKenney
Quoted 5 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Uladzislau Rezki (Sony)
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 7 lines
Quoted 3 lines
I do not see any use of internal slab interfaces by this code. It seems to be using rcu internals though. So it would best be placed with the rcu code.
That is indeed the current state. The point of moving it is to later take advantage of internal slab state.
And, in fact we already have some integrations. For example a barrier has been added for slab caches.
--
Uladzislau Rezki
Quoted 9 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
https://lore.kernel.org/linux-mm/[email protected]/T/
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
Also how about a followup patch moving the rcu-tiny implementation of kvfree_call_rcu()?
We might also consider moving the kfree_rcu*() entry points from rcupdate.h to slab.h, what do you think, is it a more logical place for them? There's some risk that files that include rcupdate.h and not slab.h would break, so that will need some build testing...
Thanks,
Vlastimil
Quoted 13 lines
Uladzislau Rezki (Sony) (5):
rcu/kvfree: Initializekvfree_rcu()separately rcu/kvfree: Move some functions underCONFIG_TINY_RCUrcu/kvfree: Adjust names passed into trace functions rcu/kvfree: Adjust a shrinker name
mm/slab: Movekvfree_rcu()into SLABinclude/linux/slab.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 876 ------------------------------------------ mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 882 insertions(+), 876 deletions(-)
Paul E. McKenney
maintainerQuoted 17 lines · Vlastimil Babka
Quoted 9 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
https://lore.kernel.org/linux-mm/[email protected]/T/
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
Quoted 7 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?We might also consider moving the kfree_rcu*() entry points from
rcupdate.htoslab.h, what do you think, is it a more logical place for them? There's some risk that files that includercupdate.hand notslab.hwould break, so that will need some build testing...
Moving the RCU Tiny implemention (or maybe even just retiring it in favor of the RCU Tree implementation) and moving the entry points make sense to me!
Thanx, Paul
Quoted 17 lines
Thanks,
VlastimilQuoted 13 lines
Uladzislau Rezki (Sony) (5):
rcu/kvfree: Initializekvfree_rcu()separately rcu/kvfree: Move some functions underCONFIG_TINY_RCUrcu/kvfree: Adjust names passed into trace functions rcu/kvfree: Adjust a shrinker name
mm/slab: Movekvfree_rcu()into SLABinclude/linux/slab.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 876 ------------------------------------------ mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 882 insertions(+), 876 deletions(-)
Uladzislau Rezki
maintainerQuoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
I will :)
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?
As, Paul already noted, it would make sense. Or just remove a tiny implementation.
Quoted 6 lines
We might also consider moving the kfree_rcu*() entry points from
rcupdate.htoslab.h, what do you think, is it a more logical place for them? There's some risk that files that includercupdate.hand notslab.hwould break, so that will need some build testing...
I agree. I have not moved them in this series, because it requires more testing due to a build break. I can work on this further, so it is not an issue.
Thank you for taking this!
--
Uladzislau Rezki
Quoted 18 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
Quoted 7 lines
I will :)
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Quoted 14 lines
Quoted 6 lines
We might also consider moving the kfree_rcu*() entry points from
rcupdate.htoslab.h, what do you think, is it a more logical place for them? There's some risk that files that includercupdate.hand notslab.hwould break, so that will need some build testing...I agree. I have not moved them in this series, because it requires more testing due to a build break. I can work on this further, so it is not an issue.
Thank you for taking this!
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 21 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
I will :)
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal kvfree_rcu() path.
After that we do not depend on CONFIG_RCU_TINY option. Probably we need also to perform some adaptation of regular kvfree_rcu() for a single CPU system.
--
Uladzislau Rezki
Quoted 31 lines · Uladzislau Rezki
Quoted 21 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
I will :)
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Quoted 7 lines
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.
Maybe we could change it to use CONFIG_SLUB_TINY as that has similar use case (less memory usage on low memory system, tradeoff for worse performance).
Quoted 6 lines
After that we do not depend on
CONFIG_RCU_TINYoption. Probably we need also to perform some adaptation of regularkvfree_rcu()for a single CPU system.--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 36 lines · Vlastimil Babka
Quoted 35 lines · Uladzislau Rezki
Quoted 21 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
I will :)
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
Quoted 7 lines
Quoted 3 lines
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).
Yep, i also was thinking about that without saying it :)
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 37 lines · Uladzislau Rezki
Quoted 36 lines · Vlastimil Babka
Quoted 35 lines · Uladzislau Rezki
Quoted 21 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
I will :)
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
Quoted 8 lines
Quoted 7 lines
Quoted 3 lines
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Thanx, Paul
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.
Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined __kvfree_call_rcu() is not a show-stopper here.
Then I wanted to switch the #ifdefs from CONFIG_TINY_RCU to CONFIG_SLUB_TINY to control whether we use the full blown batching implementation or the simple call_rcu() implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies of kvfree_rcu() on RCU internals :)
Problem 1: !CONFIG_SLUB_TINY with CONFIG_TINY_RCU
AFAICS the batching implementation includes kfree_rcu_scheduler_running() which is called from rcu_set_runtime_mode() but only on TREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in the TREE_RCU implementation
Possible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2: CONFIG_TREE_RCU with !CONFIG_SLUB_TINY
AFAICS I can't just make the simple implementation do call_rcu() on CONFIG_TREE_RCU, because call_rcu() no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see how rcu_reclaim_tiny() does that but no such equivalent exists in TREE_RCU. Am I right?
Possible solution: teach TREE_RCU callback invocation to handle __is_kvfree_rcu_offset() again, perhaps hide that branch behind #ifndef CONFIG_SLUB_TINY to avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate how kvfree_rcu() is not purely a slab thing but RCU has to special case it still.
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the rcu_head offset.
Downside: this will have some overhead, but SLUB_TINY is not meant to be performant anyway so we might not care.
Upside: we can remove __is_kvfree_rcu_offset() from TINY_RCU as well
Thoughts?
[1]
https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/linux.git/log/?h=slub-tiny-kfree_rcu
Uladzislau Rezki
maintainerQuoted 66 lines · Vlastimil Babka
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined
__kvfree_call_rcu()is not a show-stopper here.Then I wanted to switch the #ifdefs from
CONFIG_TINY_RCUtoCONFIG_SLUB_TINYto control whether we use the full blown batching implementation or the simplecall_rcu()implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies ofkvfree_rcu()on RCU internals :)Problem 1: !CONFIG_SLUB_TINY with
CONFIG_TINY_RCUAFAICS the batching implementation includes
kfree_rcu_scheduler_running()which is called fromrcu_set_runtime_mode()but only onTREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in theTREE_RCUimplementationPossible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2:
CONFIG_TREE_RCUwith !CONFIG_SLUB_TINYAFAICS I can't just make the simple implementation do
call_rcu()onCONFIG_TREE_RCU, becausecall_rcu()no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see howrcu_reclaim_tiny()does that but no such equivalent exists inTREE_RCU. Am I right?Possible solution: teach
TREE_RCUcallback invocation to handle__is_kvfree_rcu_offset()again, perhaps hide that branch behind #ifndefCONFIG_SLUB_TINYto avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate howkvfree_rcu()is not purely a slab thing but RCU has to special case it still.
Quoted 9 lines
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the
rcu_headoffset.
Downside: this will have some overhead, butSLUB_TINYis not meant to be performant anyway so we might not care.
Upside: we can remove__is_kvfree_rcu_offset()fromTINY_RCUas wellThoughts?
For the call_rcu() and to be able to reclaim over it we need to patch the tree.c(please note TINY already works):
<snip>
kernel/rcu/tree.c +9 -4
@@ -2559,13 +2559,19 @@ static void rcu_do_batch(struct rcu_data *rdp)debug_rcu_head_unqueue (rhp);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_callback (rcu_state.name, rhp); f = rhp->func; -debug_rcu_head_callback (rhp); -WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); -f (rhp); +if (__is_kvfree_rcu_offset ((unsigned long ) f)) { +trace_rcu_invoke_kvfree_callback ("" , rhp, (unsigned long ) f); +kvfree ((void *) rhp - (unsigned long ) f); + }else { +trace_rcu_invoke_callback (rcu_state.name, rhp); +debug_rcu_head_callback (rhp); +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + }rcu_lock_release (&rcu_callback_map);/*
<snip>
Mixing up CONFIG_SLUB_TINY with CONFIG_TINY_RCU in the slab_common.c should be avoided, i.e. if we can, we should eliminate a dependency on TREE_RCU or TINY_RCU in a slab. As much as possible.
So, it requires a more closer look for sure :)
--
Uladzislau Rezki
Quoted 95 lines · Uladzislau Rezki
Quoted 66 lines · Vlastimil Babka
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined
__kvfree_call_rcu()is not a show-stopper here.Then I wanted to switch the #ifdefs from
CONFIG_TINY_RCUtoCONFIG_SLUB_TINYto control whether we use the full blown batching implementation or the simplecall_rcu()implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies ofkvfree_rcu()on RCU internals :)Problem 1: !CONFIG_SLUB_TINY with
CONFIG_TINY_RCUAFAICS the batching implementation includes
kfree_rcu_scheduler_running()which is called fromrcu_set_runtime_mode()but only onTREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in theTREE_RCUimplementationPossible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2:
CONFIG_TREE_RCUwith !CONFIG_SLUB_TINYAFAICS I can't just make the simple implementation do
call_rcu()onCONFIG_TREE_RCU, becausecall_rcu()no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see howrcu_reclaim_tiny()does that but no such equivalent exists inTREE_RCU. Am I right?Possible solution: teach
TREE_RCUcallback invocation to handle__is_kvfree_rcu_offset()again, perhaps hide that branch behind #ifndefCONFIG_SLUB_TINYto avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate howkvfree_rcu()is not purely a slab thing but RCU has to special case it still.Quoted 9 lines
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the
rcu_headoffset.
Downside: this will have some overhead, butSLUB_TINYis not meant to be performant anyway so we might not care.
Upside: we can remove__is_kvfree_rcu_offset()fromTINY_RCUas wellThoughts?
For the
call_rcu()and to be able to reclaim over it we need to patch thetree.c(please note TINY already works):<snip>
kernel/rcu/tree.c +0
-4@@ -2559,13 +2559,19 @@ static void rcu_do_batch(struct rcu_data *rdp)debug_rcu_head_unqueue (rhp);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_callback (rcu_state.name, rhp); f = rhp->func; -debug_rcu_head_callback (rhp); -WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); -f (rhp);
kernel/rcu/tree.c static void rcu_do_batch(struct rcu_data *rdp) · line 2574
Quoted 10 lines
kernel/rcu/tree.c +9
-0+if (__is_kvfree_rcu_offset ((unsigned long ) f)) { +trace_rcu_invoke_kvfree_callback ("" , rhp, (unsigned long ) f); +kvfree ((void *) rhp - (unsigned long ) f); + }else { +trace_rcu_invoke_callback (rcu_state.name, rhp); +debug_rcu_head_callback (rhp); +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + }rcu_lock_release (&rcu_callback_map);
Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking __is_kvfree_rcu_offset() even if the batching is done in slab and this function is never called with an offset.
After coming up with Possible solution 2, I've started liking the idea more as RCU could then forget about the __is_kvfree_rcu_offset() "callbacks" completely, and the performant case of TREE_RCU + batching would be unaffected.
I'm speculating perhaps if there was not CONFIG_SLOB in the past, the __is_kvfree_rcu_offset() would never exist in the first place? SLAB and SLUB both can determine start of the object from a pointer to the middle of it, while SLOB couldn't.
Quoted 8 lines
kernel/rcu/tree.c +0
-0/* Mixing up
CONFIG_SLUB_TINYwithCONFIG_TINY_RCUin theslab_common.cshould be avoided, i.e. if we can, we should eliminate a dependency onTREE_RCUorTINY_RCUin a slab. As much as possible.So, it requires a more closer look for sure :)
That requires solving Problem 1 above, but question is if it's worth the trouble. Systems running TINY_RCU are unlikely to benefit from the batching?
But sure there's also possibility to hide these dependencies in KConfig, so the slab code would only consider a single (for example) #ifdef CONFIG_KVFREE_RCU_BATCHING that would be set automatically depending on TREE_RCU and !SLUB_TINY.
Quoted 2 lines
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 96 lines · Vlastimil Babka
Quoted 95 lines · Uladzislau Rezki
Quoted 66 lines · Vlastimil Babka
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined
__kvfree_call_rcu()is not a show-stopper here.Then I wanted to switch the #ifdefs from
CONFIG_TINY_RCUtoCONFIG_SLUB_TINYto control whether we use the full blown batching implementation or the simplecall_rcu()implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies ofkvfree_rcu()on RCU internals :)Problem 1: !CONFIG_SLUB_TINY with
CONFIG_TINY_RCUAFAICS the batching implementation includes
kfree_rcu_scheduler_running()which is called fromrcu_set_runtime_mode()but only onTREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in theTREE_RCUimplementationPossible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2:
CONFIG_TREE_RCUwith !CONFIG_SLUB_TINYAFAICS I can't just make the simple implementation do
call_rcu()onCONFIG_TREE_RCU, becausecall_rcu()no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see howrcu_reclaim_tiny()does that but no such equivalent exists inTREE_RCU. Am I right?Possible solution: teach
TREE_RCUcallback invocation to handle__is_kvfree_rcu_offset()again, perhaps hide that branch behind #ifndefCONFIG_SLUB_TINYto avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate howkvfree_rcu()is not purely a slab thing but RCU has to special case it still.Quoted 9 lines
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the
rcu_headoffset.
Downside: this will have some overhead, butSLUB_TINYis not meant to be performant anyway so we might not care.
Upside: we can remove__is_kvfree_rcu_offset()fromTINY_RCUas wellThoughts?
For the
call_rcu()and to be able to reclaim over it we need to patch thetree.c(please note TINY already works):<snip>
kernel/rcu/tree.c +0
-4@@ -2559,13 +2559,19 @@ static void rcu_do_batch(struct rcu_data *rdp)debug_rcu_head_unqueue (rhp);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_callback (rcu_state.name, rhp); f = rhp->func; -debug_rcu_head_callback (rhp); -WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); -f (rhp);
Quoted 15 lines
kernel/rcu/tree.c static void rcu_do_batch(struct rcu_data *rdp) · line 2574
Quoted 10 lines
kernel/rcu/tree.c +9
-0+if (__is_kvfree_rcu_offset ((unsigned long ) f)) { +trace_rcu_invoke_kvfree_callback ("" , rhp, (unsigned long ) f); +kvfree ((void *) rhp - (unsigned long ) f); + }else { +trace_rcu_invoke_callback (rcu_state.name, rhp); +debug_rcu_head_callback (rhp); +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + }rcu_lock_release (&rcu_callback_map);Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking
__is_kvfree_rcu_offset()even if the batching is done in slab and this function is never called with an offset.
Or fulfilling a missing functionality? TREE is broken in that sense whereas a TINY handles it without any issues.
It can be called for SLUB_TINY option, just call_rcu() instead of batching layer. And yes, kvfree_rcu_barrier() switches to rcu_barrier().
Quoted 6 lines
After coming up with Possible solution 2, I've started liking the idea more as RCU could then forget about the
__is_kvfree_rcu_offset()"callbacks" completely, and the performant case ofTREE_RCU+ batching would be unaffected.
I doubt it is a performance issue :)
Quoted 6 lines
I'm speculating perhaps if there was not
CONFIG_SLOBin the past, the__is_kvfree_rcu_offset()would never exist in the first place? SLAB and SLUB both can determine start of the object from a pointer to the middle of it, while SLOB couldn't.
We needed just to reclaim over RCU. So, i do not know. Paul probably knows more then me :)
Quoted 9 lines
Quoted 8 lines
kernel/rcu/tree.c +0
-0/* Mixing up
CONFIG_SLUB_TINYwithCONFIG_TINY_RCUin theslab_common.cshould be avoided, i.e. if we can, we should eliminate a dependency onTREE_RCUorTINY_RCUin a slab. As much as possible.So, it requires a more closer look for sure :)
Quoted 8 lines
That requires solving Problem 1 above, but question is if it's worth the trouble. Systems running
TINY_RCUare unlikely to benefit from the batching?But sure there's also possibility to hide these dependencies in KConfig, so the slab code would only consider a single (for example) #ifdef
CONFIG_KVFREE_RCU_BATCHINGthat would be set automatically depending onTREE_RCUand !SLUB_TINY.
It is for small systems. We can use TINY or !SMP. We covered this AFAIR that a single CPU system should not go with batching:
#ifdef SLUB_TINY || !SMP || TINY_RCU
or:
config TINY_RCU
bool default y if !PREEMPT_RCU && !SMP
+ select SLUB_TINY
Paul, more input?
--
Uladzislau Rezki
Paul E. McKenney
maintainerQuoted 112 lines · Uladzislau Rezki
Quoted 96 lines · Vlastimil Babka
Quoted 95 lines · Uladzislau Rezki
Quoted 66 lines · Vlastimil Babka
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined
__kvfree_call_rcu()is not a show-stopper here.Then I wanted to switch the #ifdefs from
CONFIG_TINY_RCUtoCONFIG_SLUB_TINYto control whether we use the full blown batching implementation or the simplecall_rcu()implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies ofkvfree_rcu()on RCU internals :)Problem 1: !CONFIG_SLUB_TINY with
CONFIG_TINY_RCUAFAICS the batching implementation includes
kfree_rcu_scheduler_running()which is called fromrcu_set_runtime_mode()but only onTREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in theTREE_RCUimplementationPossible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2:
CONFIG_TREE_RCUwith !CONFIG_SLUB_TINYAFAICS I can't just make the simple implementation do
call_rcu()onCONFIG_TREE_RCU, becausecall_rcu()no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see howrcu_reclaim_tiny()does that but no such equivalent exists inTREE_RCU. Am I right?Possible solution: teach
TREE_RCUcallback invocation to handle__is_kvfree_rcu_offset()again, perhaps hide that branch behind #ifndefCONFIG_SLUB_TINYto avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate howkvfree_rcu()is not purely a slab thing but RCU has to special case it still.Quoted 9 lines
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the
rcu_headoffset.
Downside: this will have some overhead, butSLUB_TINYis not meant to be performant anyway so we might not care.
Upside: we can remove__is_kvfree_rcu_offset()fromTINY_RCUas wellThoughts?
For the
call_rcu()and to be able to reclaim over it we need to patch thetree.c(please note TINY already works):<snip>
kernel/rcu/tree.c +0
-4@@ -2559,13 +2559,19 @@ static void rcu_do_batch(struct rcu_data *rdp)debug_rcu_head_unqueue (rhp);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_callback (rcu_state.name, rhp); f = rhp->func; -debug_rcu_head_callback (rhp); -WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); -f (rhp);Quoted 15 lines
kernel/rcu/tree.c static void rcu_do_batch(struct rcu_data *rdp) · line 2574
Quoted 10 lines
kernel/rcu/tree.c +9
-0+if (__is_kvfree_rcu_offset ((unsigned long ) f)) { +trace_rcu_invoke_kvfree_callback ("" , rhp, (unsigned long ) f); +kvfree ((void *) rhp - (unsigned long ) f); + }else { +trace_rcu_invoke_callback (rcu_state.name, rhp); +debug_rcu_head_callback (rhp); +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + }rcu_lock_release (&rcu_callback_map);Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking
__is_kvfree_rcu_offset()even if the batching is done in slab and this function is never called with an offset.
Quoted 5 lines
Or fulfilling a missing functionality? TREE is broken in that sense whereas a TINY handles it without any issues.
It can be called for
SLUB_TINYoption, justcall_rcu()instead of batching layer. And yes,kvfree_rcu_barrier()switches torcu_barrier().
Would this make sense?
if (IS_ENABLED(CONFIG_TINY_RCU) && __is_kvfree_rcu_offset((unsigned long) f)) {Just to be repetitive, other alternatives include:
1. Take advantage of SLOB being no longer with us.
2. Get rid of Tiny RCU's special casing of kfree_rcu(), and then
eliminate the above "if" statement in favor of its "else" clause.
3. Make Tiny RCU implement a trivial version of kfree_rcu() that
passes a list through RCU.
I don't have strong feelings, and am happy to defer to your guys' decision.
Quoted 6 lines
Quoted 5 lines
After coming up with Possible solution 2, I've started liking the idea more as RCU could then forget about the
__is_kvfree_rcu_offset()"callbacks" completely, and the performant case ofTREE_RCU+ batching would be unaffected.I doubt it is a performance issue :)
Me neither, especially with IS_ENABLED().
Quoted 7 lines
Quoted 5 lines
I'm speculating perhaps if there was not
CONFIG_SLOBin the past, the__is_kvfree_rcu_offset()would never exist in the first place? SLAB and SLUB both can determine start of the object from a pointer to the middle of it, while SLOB couldn't.We needed just to reclaim over RCU. So, i do not know. Paul probably knows more then me :)
In the absence of SLOB, yes, I would hope that I would have thought of determining the start of the object from a pointer to the middle of it. Or that someone would have pointed that out during review. But I honestly do not remember. ;-)
Quoted 30 lines
Quoted 9 lines
Quoted 8 lines
kernel/rcu/tree.c +0
-0/* Mixing up
CONFIG_SLUB_TINYwithCONFIG_TINY_RCUin theslab_common.cshould be avoided, i.e. if we can, we should eliminate a dependency onTREE_RCUorTINY_RCUin a slab. As much as possible.So, it requires a more closer look for sure :)
Quoted 8 lines
That requires solving Problem 1 above, but question is if it's worth the trouble. Systems running
TINY_RCUare unlikely to benefit from the batching?But sure there's also possibility to hide these dependencies in KConfig, so the slab code would only consider a single (for example) #ifdef
CONFIG_KVFREE_RCU_BATCHINGthat would be set automatically depending onTREE_RCUand !SLUB_TINY.It is for small systems. We can use TINY or !SMP. We covered this AFAIR that a single CPU system should not go with batching:
#ifdef
SLUB_TINY|| !SMP ||TINY_RCUor:
config
TINY_RCUbool default y if !PREEMPT_RCU && !SMP(quoted diff) +1
-0+ select SLUB_TINY
Quoted 1 line
Paul, more input?
I will say that Tiny RCU used to get much more focus from its users 10-15 years ago than it does now. So one approach is to implement the simplest option, and add any needed complexity back in when and if people complain. ;-)
Thanx, Paul
Quoted 128 lines · Paul E. McKenney
Quoted 112 lines · Uladzislau Rezki
Quoted 96 lines · Vlastimil Babka
Quoted 95 lines · Uladzislau Rezki
Quoted 66 lines · Vlastimil Babka
Quoted 24 lines · Paul E. McKenney
Quoted 23 lines · Uladzislau Rezki
Quoted 22 lines · Vlastimil Babka
Quoted 20 lines · Uladzislau Rezki
Quoted 12 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki
Quoted 3 lines · Vlastimil Babka
Also how about a followup patch moving the rcu-tiny implementation of
kvfree_call_rcu()?As, Paul already noted, it would make sense. Or just remove a tiny implementation.
AFAICS tiny rcu is for !SMP systems. Do they benefit from the "full" implementation with all the batching etc or would that be unnecessary overhead?
Yes, it is for a really small systems with low amount of memory. I see only one overhead it is about driving objects in pages. For a small system it can be critical because we allocate.
From the other hand, for a tiny variant we can modify the normal variant by bypassing batching logic, thus do not consume memory(for Tiny case) i.e. merge it to a normal
kvfree_rcu()path.Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Maybe we could change it to use
CONFIG_SLUB_TINYas that has similar use case (less memory usage on low memory system, tradeoff for worse performance).Yep, i also was thinking about that without saying it :)
Works for me as well!
Hi, so I tried looking at this. First I just moved the code to slab as seen in the top-most commit here [1]. Hope the non-inlined
__kvfree_call_rcu()is not a show-stopper here.Then I wanted to switch the #ifdefs from
CONFIG_TINY_RCUtoCONFIG_SLUB_TINYto control whether we use the full blown batching implementation or the simplecall_rcu()implmentation, and realized it's not straightforward and reveals there are still some subtle dependencies ofkvfree_rcu()on RCU internals :)Problem 1: !CONFIG_SLUB_TINY with
CONFIG_TINY_RCUAFAICS the batching implementation includes
kfree_rcu_scheduler_running()which is called fromrcu_set_runtime_mode()but only onTREE_RCU. Perhaps there are other facilities the batching implementation needs that only exists in theTREE_RCUimplementationPossible solution: batching implementation depends on both !CONFIG_SLUB_TINY and !CONFIG_TINY_RCU. I think it makes sense as both !SMP systems and small memory systems are fine with the simple implementation.
Problem 2:
CONFIG_TREE_RCUwith !CONFIG_SLUB_TINYAFAICS I can't just make the simple implementation do
call_rcu()onCONFIG_TREE_RCU, becausecall_rcu()no longer knows how to handle the fake callback (__is_kvfree_rcu_offset()) - I see howrcu_reclaim_tiny()does that but no such equivalent exists inTREE_RCU. Am I right?Possible solution: teach
TREE_RCUcallback invocation to handle__is_kvfree_rcu_offset()again, perhaps hide that branch behind #ifndefCONFIG_SLUB_TINYto avoid overhead if the batching implementation is used.
Downside: we visibly demonstrate howkvfree_rcu()is not purely a slab thing but RCU has to special case it still.Quoted 9 lines
Possible solution 2: instead of the special offset handling, SLUB provides a callback function, which will determine pointer to the object from the pointer to a middle of it without knowing the
rcu_headoffset.
Downside: this will have some overhead, butSLUB_TINYis not meant to be performant anyway so we might not care.
Upside: we can remove__is_kvfree_rcu_offset()fromTINY_RCUas wellThoughts?
For the
call_rcu()and to be able to reclaim over it we need to patch thetree.c(please note TINY already works):<snip>
kernel/rcu/tree.c +0
-4@@ -2559,13 +2559,19 @@ static void rcu_do_batch(struct rcu_data *rdp)debug_rcu_head_unqueue (rhp);rcu_lock_acquire (&rcu_callback_map); -trace_rcu_invoke_callback (rcu_state.name, rhp); f = rhp->func; -debug_rcu_head_callback (rhp); -WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); -f (rhp);Quoted 15 lines
kernel/rcu/tree.c static void rcu_do_batch(struct rcu_data *rdp) · line 2574
Quoted 10 lines
kernel/rcu/tree.c +9
-0+if (__is_kvfree_rcu_offset ((unsigned long ) f)) { +trace_rcu_invoke_kvfree_callback ("" , rhp, (unsigned long ) f); +kvfree ((void *) rhp - (unsigned long ) f); + }else { +trace_rcu_invoke_callback (rcu_state.name, rhp); +debug_rcu_head_callback (rhp); +WRITE_ONCE (rhp->func, (rcu_callback_t )0L ); +f (rhp); + }rcu_lock_release (&rcu_callback_map);Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking
__is_kvfree_rcu_offset()even if the batching is done in slab and this function is never called with an offset.Quoted 5 lines
Or fulfilling a missing functionality? TREE is broken in that sense whereas a TINY handles it without any issues.
It can be called for
SLUB_TINYoption, justcall_rcu()instead of batching layer. And yes,kvfree_rcu_barrier()switches torcu_barrier().Would this make sense?
if (IS_ENABLED(CONFIG_TINY_RCU) && __is_kvfree_rcu_offset((unsigned long) f)) {Just to be repetitive, other alternatives include:
1. Take advantage of SLOB being no longer with us.
Quoted 8 lines
2. Get rid of Tiny RCU's special casing of
kfree_rcu(), and then
eliminate the above "if" statement in favor of its "else" clause.3. Make Tiny RCU implement a trivial version of
kfree_rcu()that
passes a list through RCU.I don't have strong feelings, and am happy to defer to your guys' decision.
If I may chime in with an opinion, I think the cleanest approach would be to not special-case the func pointer and instead provide a callback from the SLAB layer which does the kfree. Then get rid of __is_kvfree_rcu_offset() and its usage from Tiny. Granted, there is the overhead of function calling, but I highly doubt that it is going to be a bottleneck, considering that the __is_kvfree_rcu_offset() path is a kfree slow-path. I feel in the long run, this will also be more maintainable.
Or is there a reason other than the theoretical function call overhead why this may not work?
thanks,
- Joel
Quoted 28 lines · Joel Fernandes
Quoted 27 lines · Paul E. McKenney
Quoted 10 lines · Uladzislau Rezki
Quoted 4 lines · Vlastimil Babka
Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking
__is_kvfree_rcu_offset()even if the batching is done in slab and this function is never called with an offset.Or fulfilling a missing functionality? TREE is broken in that sense whereas a TINY handles it without any issues.
It can be called for
SLUB_TINYoption, justcall_rcu()instead of batching layer. And yes,kvfree_rcu_barrier()switches torcu_barrier().Would this make sense?
if (IS_ENABLED(CONFIG_TINY_RCU) && __is_kvfree_rcu_offset((unsigned long) f)) {Just to be repetitive, other alternatives include:
1. Take advantage of SLOB being no longer with us.
2. Get rid of Tiny RCU's special casing of
kfree_rcu(), and then
eliminate the above "if" statement in favor of its "else" clause.3. Make Tiny RCU implement a trivial version of
kfree_rcu()that
passes a list through RCU.
Quoted 6 lines
Quoted 2 lines
I don't have strong feelings, and am happy to defer to your guys' decision.
If I may chime in with an opinion, I think the cleanest approach would be to not special-case the func pointer and instead provide a callback from the SLAB layer which does the kfree. Then get rid of
Right.
Quoted 8 lines
__is_kvfree_rcu_offset()and its usage from Tiny. Granted, there is the overhead of function calling, but I highly doubt that it is going to be a bottleneck, considering that the__is_kvfree_rcu_offset()path is a kfree slow-path. I feel in the long run, this will also be more maintainable.Or is there a reason other than the theoretical function call overhead why this may not work?
My concern was about the overhead of calculating the pointer to the object starting address, but it's just some arithmetics, so it should be negligible. So I'm prototyping this approach now. Thanks all.
Quoted 3 lines
thanks,
- Joel
On Wed, Jan 22, 2025 at 11:43 AM Vlastimil Babka <[email protected]> wrote:
[...]
Quoted 6 lines
Quoted 6 lines
__is_kvfree_rcu_offset()and its usage from Tiny. Granted, there is the overhead of function calling, but I highly doubt that it is going to be a bottleneck, considering that the__is_kvfree_rcu_offset()path is a kfree slow-path. I feel in the long run, this will also be more maintainable.
Quoted 6 lines
Quoted 2 lines
Or is there a reason other than the theoretical function call overhead why this may not work?
My concern was about the overhead of calculating the pointer to the object starting address, but it's just some arithmetics, so it should be negligible. So I'm prototyping this approach now. Thanks all.
Ah, that's a valid point. Looking forward to reviewing the patch and hope it works out!
thanks,
- Joel
Uladzislau Rezki
maintainerQuoted 44 lines · Vlastimil Babka
Quoted 28 lines · Joel Fernandes
Quoted 27 lines · Paul E. McKenney
Quoted 10 lines · Uladzislau Rezki
Quoted 4 lines · Vlastimil Babka
Right so that's the first Possible solution, but without the #ifdef. So there's an overhead of checking
__is_kvfree_rcu_offset()even if the batching is done in slab and this function is never called with an offset.Or fulfilling a missing functionality? TREE is broken in that sense whereas a TINY handles it without any issues.
It can be called for
SLUB_TINYoption, justcall_rcu()instead of batching layer. And yes,kvfree_rcu_barrier()switches torcu_barrier().Would this make sense?
if (IS_ENABLED(CONFIG_TINY_RCU) && __is_kvfree_rcu_offset((unsigned long) f)) {Just to be repetitive, other alternatives include:
1. Take advantage of SLOB being no longer with us.
2. Get rid of Tiny RCU's special casing of
kfree_rcu(), and then
eliminate the above "if" statement in favor of its "else" clause.3. Make Tiny RCU implement a trivial version of
kfree_rcu()that
passes a list through RCU.Quoted 6 lines
Quoted 2 lines
I don't have strong feelings, and am happy to defer to your guys' decision.
If I may chime in with an opinion, I think the cleanest approach would be to not special-case the func pointer and instead provide a callback from the SLAB layer which does the kfree. Then get rid of
Right.
Quoted 6 lines
__is_kvfree_rcu_offset()and its usage from Tiny. Granted, there is the overhead of function calling, but I highly doubt that it is going to be a bottleneck, considering that the__is_kvfree_rcu_offset()path is a kfree slow-path. I feel in the long run, this will also be more maintainable.
Quoted 7 lines
Quoted 2 lines
Or is there a reason other than the theoretical function call overhead why this may not work?
My concern was about the overhead of calculating the pointer to the object starting address, but it's just some arithmetics, so it should be negligible. So I'm prototyping this approach now. Thanks all.
You are welcome :)
--
Uladzislau Rezki
Uladzislau Rezki
maintainerQuoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
<snip>
mm/slab: Move kvfree_rcu() into SLAB
Move kvfree_rcu() functionality to the slab_common.c file.
The reason of being kvfree_rcu() functionality as part of SLAB is that, there is a clear trend and need of closer integration. One of the recent example is creating a barrier function for SLAB caches.
Another reason is to prevent of having several implementations of RCU machinery for reclaiming objects after a GP. As future steps, it can be more integrated(easier) with SLAB internals. <snip>
--
Uladzislau Rezki
Quoted 23 lines · Uladzislau Rezki
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
Quoted 9 lines
Quoted 2 lines
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Looks good, will include in slab/for-next
I think patch 5 should add more explanation to the commit message - the subthread started by Christoph could provide content :) Can you summarize so I can amend the commit log?
<snip>
mm/slab: Movekvfree_rcu()into SLABMove
kvfree_rcu()functionality to theslab_common.cfile.
Quoted 8 lines
The reason of being
kvfree_rcu()functionality as part of SLAB is that, there is a clear trend and need of closer integration. One of the recent example is creating a barrier function for SLAB caches.Another reason is to prevent of having several implementations of RCU machinery for reclaiming objects after a GP. As future steps, it can be more integrated(easier) with SLAB internals. <snip>
Thanks, amended.
Quoted 2 lines
--
Uladzislau Rezki
Quoted 17 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
https://lore.kernel.org/linux-mm/[email protected]/T/
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Uladzislau Rezki (Sony) (5):
rcu/kvfree: Initializekvfree_rcu()separately rcu/kvfree: Move some functions underCONFIG_TINY_RCUrcu/kvfree: Adjust names passed into trace functions rcu/kvfree: Adjust a shrinker name
mm/slab: Movekvfree_rcu()into SLAB
Quoted 5 lines
include/linux/slab.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 876 ------------------------------------------ mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 882 insertions(+), 876 deletions(-)
Sorry for the late reply, but better late than never...
FWIW,
Thanks for all the efforts!
By the way, any future plans how to take advantage of internal slab state?
--
Hyeonggon
Vlastimil Babka
AppliedQuoted 26 lines · Hyeonggon Yoo
Quoted 17 lines · Uladzislau Rezki (Sony)
Hello!
This is v2. It is based on the Linux 6.13-rc2. The first version is here:
https://lore.kernel.org/linux-mm/[email protected]/T/
The difference between v1 and v2 is that, the preparation process is done in original place instead and after that there is one final move.
Uladzislau Rezki (Sony) (5):
rcu/kvfree: Initializekvfree_rcu()separately rcu/kvfree: Move some functions underCONFIG_TINY_RCUrcu/kvfree: Adjust names passed into trace functions rcu/kvfree: Adjust a shrinker name
mm/slab: Movekvfree_rcu()into SLABQuoted 5 lines
include/linux/slab.h | 1 + init/main.c | 1 + kernel/rcu/tree.c | 876 ------------------------------------------ mm/slab_common.c | 880 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 882 insertions(+), 876 deletions(-)
Quoted 6 lines
Sorry for the late reply, but better late than never...
FWIW,
Acked-by Hyeonggon Yoo <[email protected]>Tested-by Hyeonggon Yoo <[email protected]>
Thanks, applied.
Quoted 4 lines
Thanks for all the efforts!
By the way, any future plans how to take advantage of internal slab state?
One way is the sheaves effort which in the last RFC also replaced the way kfree_rcu() is handled for caches with enabled sheaves.
Perhaps even without those we could consider e.g. doing the kfree_rcu() batching by grouping the pending frees per kmem cache, which could perhaps lead to more efficient flushing later.
Quoted 2 lines
--
Hyeonggon