fork: free vmapped stacks in cache when cpus are offline
Subsystems (3)
PATCH v4 1/1 fork: free vmapped stacks in cache when cpus are offline
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the cached stacks may remain unfreed even when cpu are offline.
By adding a cpu hotplug callback to free the cached stacks when a cpu
goes offline, the pages of the cached stacks are not wasted.
---
v4:
use CPUHP_BP_PREPARE_DYN state for cpuhp setup
fix minor coding style
v3:
fix misuse of per-cpu api
fix location of function definition within CONFIG_VMAP_STACK
v2:
remove cpuhp callback for startup, only teardown callback is installed.
kernel/fork.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
kernel/fork.c +23 -0
@@ -168,6 +168,24 @@ void __weak arch_release_thread_stack(unsigned long *stack)*/ #define NR_CACHED_STACKS 2 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_struct **cached_vm_stacks =per_cpu_ptr (cached_stacks, cpu); +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack = cached_vm_stacks[i]; + +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); + cached_vm_stacks[i] =NULL ; + } + +return 0 ; +}#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node)
@@ -456,6 +474,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"fork:vmstack_cache" , +NULL , free_vm_stack_cache); +#endif } int __weak arch_dup_task_struct(struct task_struct *dst,
Signature · 1 line
2.7.4
Quoted 8 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the cached stacks may remain unfreed even when cpu are offline.
By adding a cpu hotplug callback to free the cached stacks when a cpu
goes offline, the pages of the cached stacks are not wasted.Signed-off-by Hoeun Ryu <[email protected]>
Quoted 57 lines
---
v4:
useCPUHP_BP_PREPARE_DYNstate for cpuhp setup
fix minor coding style
v3:
fix misuse of per-cpu api
fix location of function definition withinCONFIG_VMAP_STACK
v2:
remove cpuhp callback forstartup, onlyteardowncallback is installed.kernel/fork.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)kernel/fork.c +23
-0@@ -168,6 +168,24 @@ void __weak arch_release_thread_stack(unsigned long *stack)*/ #define NR_CACHED_STACKS 2 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); + +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_struct **cached_vm_stacks =per_cpu_ptr (cached_stacks, cpu); +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack = cached_vm_stacks[i]; + +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); + cached_vm_stacks[i] =NULL ; + } + +return 0 ; +}#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node)@@ -456,6 +474,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"fork:vmstack_cache" , +NULL , free_vm_stack_cache); +#endif } int __weak arch_dup_task_struct(struct task_struct *dst,Signature · 1 line
2.7.4
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 24 lines · Hoeun Ryu
#define
NR_CACHED_STACKS2
staticDEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);kernel/fork.c +18
-0+ +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_struct **cached_vm_stacks =per_cpu_ptr (cached_stacks, cpu); +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack = cached_vm_stacks[i]; + +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); + cached_vm_stacks[i] =NULL ; + } + +return 0 ; +}#endif static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node)
kernel/fork.c void __init fork_init(void) · line 481
Quoted 8 lines
kernel/fork.c +5
-0@@ -456,6 +474,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"fork:vmstack_cache" , +NULL , free_vm_stack_cache); +#endif
If you wrap that in vm_stack_cache_init() and have a stub for !VMAP_STACK (add an #else to the #ifdef VMAP_STACK section above) then you spare the ugly ifdef in fork_init()
Thanks,
tglx
Quoted 26 lines · Thomas Gleixner
Quoted 24 lines · Hoeun Ryu
#define
NR_CACHED_STACKS2
staticDEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);kernel/fork.c +18
-0+ +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_struct **cached_vm_stacks =per_cpu_ptr (cached_stacks, cpu); +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack = cached_vm_stacks[i]; + +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); + cached_vm_stacks[i] =NULL ; + } + +return 0 ; +}#endif
Quoted 14 lines
kernel/fork.c void __init fork_init(void) · line 481
Quoted 10 lines
static unsigned long *alloc_thread_stack_node(struct
task_struct*tsk, int node)kernel/fork.c +5
-0@@ -456,6 +474,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"fork:vmstack_cache" , +NULL , free_vm_stack_cache); +#endif If you wrap that in
vm_stack_cache_init()and have a stub for !VMAP_STACK (add an #else to the #ifdefVMAP_STACKsection above) then you spare the ugly ifdef infork_init()
OK. I will.
Quoted 3 lines
Thanks,
tglx
Quoted 41 lines · Hoeun Ryu
Quoted 26 lines · Thomas Gleixner
Quoted 24 lines · Hoeun Ryu
#define
NR_CACHED_STACKS2
staticDEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);kernel/fork.c +18
-0+ +static int free_vm_stack_cache (unsigned int cpu) +{ +struct vm_struct **cached_vm_stacks =per_cpu_ptr (cached_stacks, cpu); +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack = cached_vm_stacks[i]; + +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); + cached_vm_stacks[i] =NULL ; + } + +return 0 ; +}#endif
Quoted 14 lines
kernel/fork.c void __init fork_init(void) · line 481
Quoted 10 lines
static unsigned long *alloc_thread_stack_node(struct
task_struct*tsk, int node)kernel/fork.c +5
-0@@ -456,6 +474,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_BP_PREPARE_DYN ,"fork:vmstack_cache" , +NULL , free_vm_stack_cache); +#endif If you wrap that in
vm_stack_cache_init()and have a stub for !VMAP_STACK (add an #else to the #ifdefVMAP_STACKsection above) then you spare the ugly ifdef infork_init()
Quoted 2 lines
OK. I will.
I sent v5 patch having those stub, (Kbuild error anyway).
But, I didn't consider THREAD_SIZE >= PAGE_SIZE.
so the blank inlined version of vm_stack_cache_init() should be defined twice, one for THREAD_SIZE < PAGE_SIZE && !CONFIG_VMAP_STACK, one for !CONFIG_VMAP_STACK.
Do you really want this?
---
# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK))
# if CONFIG_VMAP_STACK
int free_vm_stack_cache()
{
...
}
int vm_stack_cache_init()
{
cpuhp_setup_state(free_vm_stack_cache);
}
#else // !CONFIG_VMAP_STACK
inline int vm_stack_cache_init() { return 0; }
#endif
#else // THREAD_SIZE < PAGE_SIZE && !CONFIG_VMAP_STACK
...
...
inline int vm_stack_cache_init() { return 0; }
#endifQuoted 3 lines
Quoted 3 lines
Thanks,
tglx