fork: dynamically allocate cache array for vmapped stacks using cpuhp
Subsystems (3)
PATCH 1/3 fork: dynamically allocate cache array for vmapped stacks using cpuhp
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established in fork_init().
--- kernel/fork.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 17 deletions(-)
kernel/fork.c +64 -17
@@ -167,26 +167,71 @@ void __weak arch_release_thread_stack(unsigned long *stack)* flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 -static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +struct vm_stack_cache { +struct vm_struct **vm_stacks; +int nr; +int cur; +}; + +static DEFINE_PER_CPU(struct vm_stack_cache, vm_stacks); + +static int alloc_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +/* if free_vm_stack_cache() didn't free it */ +if (!vm_stacks) { + vm_stacks = +vzalloc (sizeof (struct vm_struct *) *NR_CACHED_STACKS ); +if (!vm_stacks) +return -ENOMEM ; + } + + vm_stack_cache->vm_stacks = vm_stacks; + vm_stack_cache->cur =0 ; + vm_stack_cache->nr =0 ; + +return 0 ; +} + +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +for (i =0 ; i < vm_stack_cache->nr; i++) { +vfree (vm_stacks[i]->addr); + vm_stacks[i] =NULL ; + } + + vm_stack_cache->nr =0 ; + vm_stack_cache->cur =0 ; +/* do not free vm_stack[cpu]->vm_stacks itself, reused in allocation */ + +return 0 ; +} +#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node) {#ifdef CONFIG_VMAP_STACK +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;void *stack; -int i;local_irq_disable (); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -struct vm_struct *s =this_cpu_read (cached_stacks[i]); - -if (!s) -continue ; -this_cpu_write (cached_stacks[i],NULL ); - - tsk->stack_vm_area = s; +if (vm_stack_cache->cur >0 ) { +struct vm_struct *vm_stack = vm_stacks[--vm_stack_cache->cur]; + tsk->stack_vm_area = vm_stack;local_irq_enable (); -return s->addr; + +return vm_stack->addr; }local_irq_enable ();
@@ -216,15 +261,14 @@ static inline void free_thread_stack(struct task_struct *tsk) {#ifdef CONFIG_VMAP_STACK if (task_stack_vm_area (tsk)) { +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;unsigned long flags; -int i;local_irq_save (flags); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -if (this_cpu_read (cached_stacks[i])) -continue ; - -this_cpu_write (cached_stacks[i], tsk->stack_vm_area); +if (vm_stack_cache->cur < vm_stack_cache->nr) { + vm_stacks[vm_stack_cache->cur++] = tsk->stack_vm_area;local_irq_restore (flags);return ; }
@@ -456,6 +500,9 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +cpuhp_setup_state (CPUHP_AP_ONLINE_DYN ,"vm_stack_cache" , + alloc_vm_stack_cache, free_vm_stack_cache); } int __weak arch_dup_task_struct(struct task_struct *dst,
Signature · 1 line
2.7.4
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
Quoted 126 lines
Signed-off-by Hoeun Ryu <[email protected]>--- kernel/fork.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 17 deletions(-)kernel/fork.c +64
-17@@ -167,26 +167,71 @@ void __weak arch_release_thread_stack(unsigned long *stack)* flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 -static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +struct vm_stack_cache { +struct vm_struct **vm_stacks; +int nr; +int cur; +}; + +static DEFINE_PER_CPU(struct vm_stack_cache, vm_stacks); + +static int alloc_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +/* if free_vm_stack_cache() didn't free it */ +if (!vm_stacks) { + vm_stacks = +vzalloc (sizeof (struct vm_struct *) *NR_CACHED_STACKS ); +if (!vm_stacks) +return -ENOMEM ; + } + + vm_stack_cache->vm_stacks = vm_stacks; + vm_stack_cache->cur =0 ; + vm_stack_cache->nr =0 ; + +return 0 ; +} + +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +for (i =0 ; i < vm_stack_cache->nr; i++) { +vfree (vm_stacks[i]->addr); + vm_stacks[i] =NULL ; + } + + vm_stack_cache->nr =0 ; + vm_stack_cache->cur =0 ; +/* do not free vm_stack[cpu]->vm_stacks itself, reused in allocation */ + +return 0 ; +} +#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node) {#ifdef CONFIG_VMAP_STACK +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;void *stack; -int i;local_irq_disable (); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -struct vm_struct *s =this_cpu_read (cached_stacks[i]); - -if (!s) -continue ; -this_cpu_write (cached_stacks[i],NULL ); - - tsk->stack_vm_area = s; +if (vm_stack_cache->cur >0 ) { +struct vm_struct *vm_stack = vm_stacks[--vm_stack_cache->cur]; + tsk->stack_vm_area = vm_stack;local_irq_enable (); -return s->addr; + +return vm_stack->addr; }local_irq_enable ();@@ -216,15 +261,14 @@ static inline void free_thread_stack(struct task_struct *tsk) {#ifdef CONFIG_VMAP_STACK if (task_stack_vm_area (tsk)) { +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;unsigned long flags; -int i;local_irq_save (flags); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -if (this_cpu_read (cached_stacks[i])) -continue ; - -this_cpu_write (cached_stacks[i], tsk->stack_vm_area); +if (vm_stack_cache->cur < vm_stack_cache->nr) { + vm_stacks[vm_stack_cache->cur++] = tsk->stack_vm_area;local_irq_restore (flags);return ; }@@ -456,6 +500,9 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +cpuhp_setup_state (CPUHP_AP_ONLINE_DYN ,"vm_stack_cache" , + alloc_vm_stack_cache, free_vm_stack_cache); } int __weak arch_dup_task_struct(struct task_struct *dst,Signature · 1 line
2.7.4
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().
Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Quoted 130 lines
Quoted 124 lines
Signed-off-by Hoeun Ryu <[email protected]>--- kernel/fork.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 17 deletions(-)kernel/fork.c +64
-18@@ -167,26 +167,71 @@ void __weak arch_release_thread_stack(unsigned long *stack)* flush. Try to minimize the number of calls by caching stacks. */ #define NR_CACHED_STACKS 2 -static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +struct vm_stack_cache { +struct vm_struct **vm_stacks; +int nr; +int cur; +}; + +static DEFINE_PER_CPU(struct vm_stack_cache, vm_stacks); + +static int alloc_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +/* if free_vm_stack_cache() didn't free it */ +if (!vm_stacks) { + vm_stacks = +vzalloc (sizeof (struct vm_struct *) *NR_CACHED_STACKS ); +if (!vm_stacks) +return -ENOMEM ; + } + + vm_stack_cache->vm_stacks = vm_stacks; + vm_stack_cache->cur =0 ; + vm_stack_cache->nr =0 ; + +return 0 ; +} + +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_stack_cache *vm_stack_cache = &per_cpu (vm_stacks, cpu); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks; +int i; + +for (i =0 ; i < vm_stack_cache->nr; i++) { +vfree (vm_stacks[i]->addr); + vm_stacks[i] =NULL ; + } + + vm_stack_cache->nr =0 ; + vm_stack_cache->cur =0 ; +/* do not free vm_stack[cpu]->vm_stacks itself, reused in allocation */ + +return 0 ; +} +#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node) {#ifdef CONFIG_VMAP_STACK +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;void *stack; -int i;local_irq_disable (); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -struct vm_struct *s =this_cpu_read (cached_stacks[i]); - -if (!s) -continue ; -this_cpu_write (cached_stacks[i],NULL ); - - tsk->stack_vm_area = s; +if (vm_stack_cache->cur >0 ) { +struct vm_struct *vm_stack = vm_stacks[--vm_stack_cache->cur]; + tsk->stack_vm_area = vm_stack;local_irq_enable (); -return s->addr; + +return vm_stack->addr; }local_irq_enable ();@@ -216,15 +261,14 @@ static inline void free_thread_stack(struct task_struct *tsk) {#ifdef CONFIG_VMAP_STACK if (task_stack_vm_area (tsk)) { +struct vm_stack_cache *vm_stack_cache = + &per_cpu (vm_stacks,smp_processor_id ()); +struct vm_struct **vm_stacks = vm_stack_cache->vm_stacks;unsigned long flags; -int i;local_irq_save (flags); -for (i =0 ; i <NR_CACHED_STACKS ; i++) { -if (this_cpu_read (cached_stacks[i])) -continue ; - -this_cpu_write (cached_stacks[i], tsk->stack_vm_area); +if (vm_stack_cache->cur < vm_stack_cache->nr) { + vm_stacks[vm_stack_cache->cur++] = tsk->stack_vm_area;local_irq_restore (flags);return ; }@@ -456,6 +500,9 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +cpuhp_setup_state (CPUHP_AP_ONLINE_DYN ,"vm_stack_cache" , + alloc_vm_stack_cache, free_vm_stack_cache); } int __weak arch_dup_task_struct(struct task_struct *dst, --Quoted 2 lines
2.7.4
--
Michal Hocko
SUSE Labs
Quoted 15 lines · Hoeun Ryu
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
Quoted 5 lines
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
Yes, that is true but cpu offline operation is just too rare for this to matter all that much I believe. More importantly, though, the current implementation could be easily fixed as well without reworking how the caching works. If there are workloads where the wastage really matters then please try to fix it with the current caching scheme before extending it for larger caches. This would make it easier to backport to older kernels.
Quoted 2 lines
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Without being more specific who might want the bigger caches and why this sounds like an insufficient justification to replace the current (simpler) caching.
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 15 lines · Hoeun Ryu
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
Quoted 7 lines
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Then how about just registering a simple hotplug hook to free the stacks without worrying about freeing the tiny array as well?
--Andy
Quoted 24 lines · Andy Lutomirski
Quoted 15 lines · Hoeun Ryu
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
Quoted 7 lines
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Quoted 3 lines
Then how about just registering a simple hotplug hook to free the stacks without worrying about freeing the tiny array as well?
Michal, What do you think about it. it sounds fair enough.
Quoted 1 line
--Andy
Quoted 25 lines · Hoeun Ryu
Quoted 24 lines · Andy Lutomirski
Quoted 22 lines · Hoeun Ryu
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Quoted 5 lines
Quoted 3 lines
Then how about just registering a simple hotplug hook to free the stacks without worrying about freeing the tiny array as well?
Michal, What do you think about it. it sounds fair enough.
This is what I've tried to suggest in the other reply.
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 26 lines · Michal Hocko
Quoted 25 lines · Hoeun Ryu
Quoted 24 lines · Andy Lutomirski
Quoted 22 lines · Hoeun Ryu
Quoted 9 lines · Michal Hocko
Quoted 7 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the array for the cached stacks is statically allocated by per-cpu api.
In this new implementation, the array for the cached stacks are dynamically
allocted and freed by cpu hotplug callbacks and the cached stacks are freed when cpu is down. setup for cpu hotplug is established infork_init().Quoted 4 lines
Why do we want this? I can see that the follow up patch makes the number configurable but the changelog doesn't describe the motivation for that. Which workload would benefit from a higher value?
The key difference of this implementation, the cached stacks for a cpu is freed when a cpu is down.
so the cached stacks are no longer wasted.
In the current implementation, the cached stacks for a cpu still remain on the system when a cpu is down.
I think we could imagine what if a machine has many cpus and someone wants to have bigger size of stack caches.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Then how about just registering a simple hotplug hook to free the stacks without worrying about freeing the tiny array as well?
Michal, What do you think about it. it sounds fair enough.
This is what I've tried to suggest in the other reply.
OK, I'll work on patch1/2 again and drop patch3.
Quoted 3 lines
--
Michal Hocko
SUSE Labs
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-dynamically-allocate-cache-array-for-vmapped-stacks-using-cpuhp/20170203-234431 config: i386-randconfig-x004-201705 (attached as .config)…
Hi Hoeun,
[auto build test ERROR on next-20170203]
[also build test ERROR on v4.10-rc6]
[cannot apply to linus/master linux/master v4.9-rc8 v4.9-rc7 v4.9-rc6]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-dynamically-allocate-cache-array-for-vmapped-stacks-using-cpuhp/20170203-234431
config: i386-randconfig-x004-201705 (attached as .config)
compiler: gcc-6 (Debian 6.2.0-3) 6.2.0 20160901 reproduce:
# save the attached .config to linux build tree
make ARCH=i386 All errors (new ones prefixed by >>):
kernel/fork.c: In function 'fork_init':
>> kernel/fork.c:505:6: error: 'alloc_vm_stack_cache' undeclared (first use in this function)
alloc_vm_stack_cache, free_vm_stack_cache);
^~~~~~~~~~~~~~~~~~~~
kernel/fork.c:505:6: note: each undeclared identifier is reported only once for each function it appears in
>> kernel/fork.c:505:28: error: 'free_vm_stack_cache' undeclared (first use in this function)
alloc_vm_stack_cache, free_vm_stack_cache);
^~~~~~~~~~~~~~~~~~~vim +/alloc_vm_stack_cache +505 kernel/fork.c
499
500 for (i = 0; i < UCOUNT_COUNTS; i++) {
501 init_user_ns.ucount_max[i] = max_threads/2;
502 }
503
504 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "vm_stack_cache",
> 505 alloc_vm_stack_cache, free_vm_stack_cache);
506 }
507
508 int __weak arch_dup_task_struct(struct task_struct *dst,
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation