fork: free vmapped stacks in cache when cpus are offline
Subsystems (3)
PATCH v2 1/2 fork: free vmapped stacks in cache when cpus are offline
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the cached stacks may remain unfreed even when cpu are offline.
By adding a cpu hotplug callback to free the cached stacks when a cpu
goes offline, the pages of the cached stacks are not wasted.
---
Changes in v2:
remove cpuhp callback for starup, only teardown callback is installed.
kernel/fork.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)
kernel/fork.c +21 -0
@@ -170,6 +170,22 @@ void __weak arch_release_thread_stack(unsigned long *stack) static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif +static int free_vm_stack_cache (unsigned int cpu) +{ +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack =this_cpu_read (cached_stacks[i]); +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); +this_cpu_write (cached_stacks[i],NULL ); + } + +return 0 ; +} +static unsigned long *alloc_thread_stack_node (struct task_struct *tsk,int node) {#ifdef CONFIG_VMAP_STACK
@@ -456,6 +472,11 @@ void __init fork_init(void)for (i =0 ; i <UCOUNT_COUNTS ; i++) { init_user_ns.ucount_max[i] = max_threads/2 ; } + +#ifdef CONFIG_VMAP_STACK +cpuhp_setup_state (CPUHP_AP_ONLINE_DYN ,"vm_stack_cache" , +NULL , free_vm_stack_cache); +#endif } int __weak arch_dup_task_struct(struct task_struct *dst,
Signature · 1 line
2.7.4
PATCH v2 2/2 fork: make number of cached stacks (vmapped) configurable using Kbuild
Introducing NR_VMAP_STACK_CACHE, the number of cached stacks for virtually mapped kernel stack can be configurable using Kbuild system. default value is 2.
--- arch/Kconfig | 8 ++++++++ kernel/fork.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-)
arch/Kconfig +8 -0
@@ -849,6 +849,14 @@ config VMAP_STACK the stack to map directly to the KASAN shadow map using a formula that is incorrect if the stack is in vmalloc space. +config NR_VMAP_STACK_CACHE + int "Number of cached stacks" + default "2" + depends on VMAP_STACK + help + This determines how many stacks can be cached for virtually + mapped kernel stacks. + config ARCH_WANT_RELAX_ORDER bool
kernel/fork.c +1 -1
@@ -166,7 +166,7 @@ void __weak arch_release_thread_stack(unsigned long *stack)* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB * flush. Try to minimize the number of calls by caching stacks. */ -#define NR_CACHED_STACKS 2 +#define NR_CACHED_STACKS CONFIG_NR_VMAP_STACK_CACHE static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif
Signature · 1 line
2.7.4
Re: [kernel-hardening] [PATCH v2 2/2] fork: make number of cached stacks (vmapped) configurable using Kbuild
arch/Kconfig config VMAP_STACK · line 859
Quoted 8 lines · Hoeun Ryu
arch/Kconfig +8
-0+config NR_VMAP_STACK_CACHE + int "Number of cached stacks" + default "2" + depends on VMAP_STACK + help + This determines how many stacks can be cached for virtually + mapped kernel stacks. +
Can you explain in the help text under what circumstances someone should change this, and to what value? As-is, it seems like somewhat of a useless config option which no one will ever change.
- Eric
Re: [PATCH v2 2/2] fork: make number of cached stacks (vmapped) configurable using Kbuild
In reply to Hoeun RyuQuoted 3 lines · Hoeun Ryu
Introducing
NR_VMAP_STACK_CACHE, the number of cached stacks for virtually mapped kernel stack can be configurable usingKbuildsystem. default value is 2.
This should really explain _why_ we want/need this. The config space is really large already and there should better be a good reason to make it even larger. Which workload will benefit from a different than default cache size and why?
Without a really good reasons NACK from me.
Quoted 41 lines
Signed-off-by Hoeun Ryu <[email protected]>--- arch/Kconfig | 8 ++++++++ kernel/fork.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-)arch/Kconfig +8
-0@@ -849,6 +849,14 @@ config VMAP_STACK the stack to map directly to the KASAN shadow map using a formula that is incorrect if the stack is in vmalloc space. +config NR_VMAP_STACK_CACHE + int "Number of cached stacks" + default "2" + depends on VMAP_STACK + help + This determines how many stacks can be cached for virtually + mapped kernel stacks. + config ARCH_WANT_RELAX_ORDER boolkernel/fork.c +1
-1@@ -166,7 +166,7 @@ void __weak arch_release_thread_stack(unsigned long *stack)* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB * flush. Try to minimize the number of calls by caching stacks. */ -#define NR_CACHED_STACKS 2 +#define NR_CACHED_STACKS CONFIG_NR_VMAP_STACK_CACHE static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif Signature · 1 line
2.7.4
Signature · 2 lines
Michal Hocko SUSE Labs
Re: [PATCH v2 2/2] fork: make number of cached stacks (vmapped) configurable using Kbuild
Quoted 5 lines · Michal Hocko
Quoted 3 lines · Hoeun Ryu
Introducing
NR_VMAP_STACK_CACHE, the number of cached stacks for virtually mapped kernel stack can be configurable usingKbuildsystem. default value is 2.
Quoted 7 lines
This should really explain _why_ we want/need this. The config space is really large already and there should better be a good reason to make it even larger. Which workload will benefit from a different than default cache size and why?
Without a really good reasons NACK from me.
OK, I understand your NACK. Thank you.
Quoted 45 lines
Quoted 39 lines
Signed-off-by Hoeun Ryu <[email protected]>--- arch/Kconfig | 8 ++++++++ kernel/fork.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-)arch/Kconfig +8
-0@@ -849,6 +849,14 @@ config VMAP_STACK the stack to map directly to the KASAN shadow map using a formula that is incorrect if the stack is in vmalloc space. +config NR_VMAP_STACK_CACHE + int "Number of cached stacks" + default "2" + depends on VMAP_STACK + help + This determines how many stacks can be cached for virtually + mapped kernel stacks. + config ARCH_WANT_RELAX_ORDER boolkernel/fork.c +1
-2@@ -166,7 +166,7 @@ void __weak arch_release_thread_stack(unsigned long *stack)* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB * flush. Try to minimize the number of calls by caching stacks. */ -#define NR_CACHED_STACKS 2 +#define NR_CACHED_STACKS CONFIG_NR_VMAP_STACK_CACHE static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif --Quoted 2 lines
2.7.4
--
Michal Hocko
SUSE Labs
Hi Hoeun,
Quoted 5 lines · Hoeun Ryu
kernel/fork.c +5
-0+static int free_vm_stack_cache (unsigned int cpu) +{ +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) {
kernel/fork.c void __weak arch_release_thread_stack(unsigned long *stack) · line 187
Quoted 10 lines
kernel/fork.c +10
-0+struct vm_struct *vm_stack =this_cpu_read (cached_stacks[i]); +if (!vm_stack) +continue ; + +vfree (vm_stack->addr); +this_cpu_write (cached_stacks[i],NULL ); + } + +return 0 ; +}
Doesn't this need to free the stacks for the 'cpu' that's passed in, instead of "this" CPU?
- Eric
Quoted 13 lines · Eric Biggers
Hi Hoeun,
Quoted 10 lines · Hoeun Ryu
kernel/fork.c +10
-0+static int free_vm_stack_cache (unsigned int cpu) +{ +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack =this_cpu_read (cached_stacks[i]); +if (!vm_stack) +continue ; + +vfree (vm_stack->addr);
Quoted 9 lines
kernel/fork.c void __weak arch_release_thread_stack(unsigned long *stack) · line 187
Quoted 5 lines
kernel/fork.c +5
-0+this_cpu_write (cached_stacks[i],NULL ); + } + +return 0 ; +}Doesn't this need to free the stacks for the 'cpu' that's passed in, instead of "this" CPU?
Sorry, Thank you for your correction. I will fix this.
Quoted 1 line
- Eric
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143 config: parisc-c3000_defconfig (attached as .config) compiler…
Hi Hoeun,
[auto build test ERROR on next-20170208]
[also build test ERROR on v4.10-rc7]
[cannot apply to linus/master linux/master v4.9-rc8 v4.9-rc7 v4.9-rc6]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143
config: parisc-c3000_defconfig (attached as .config)
compiler: hppa-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=parisc
All error/warnings (new ones prefixed by >>):
kernel/fork.c: In function 'free_vm_stack_cache':
>> kernel/fork.c:177:18: error: 'NR_CACHED_STACKS' undeclared (first use in this function)
for (i = 0; i < NR_CACHED_STACKS; i++) {
^~~~~~~~~~~~~~~~
kernel/fork.c:177:18: note: each undeclared identifier is reported only once for each function it appears in
In file included from include/asm-generic/percpu.h:6:0,
from ./arch/parisc/include/generated/asm/percpu.h:1,
from arch/parisc/include/asm/processor.h:19,
from arch/parisc/include/asm/thread_info.h:7,
from include/linux/thread_info.h:25,
from include/asm-generic/preempt.h:4,
from ./arch/parisc/include/generated/asm/preempt.h:1,
from include/linux/preempt.h:59,
from include/linux/spinlock.h:50,
from include/linux/mmzone.h:7,
from include/linux/gfp.h:5,
from include/linux/slab.h:14,
from kernel/fork.c:14:
>> kernel/fork.c:178:46: error: 'cached_stacks' undeclared (first use in this function)
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^
include/linux/percpu-defs.h:305:9: note: in definition of macro '__pcpu_size_call_return'
typeof(variable) pscr_ret__; \
^~~~~~~~
>> kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~
>> include/linux/percpu-defs.h:304:1: warning: initialization makes pointer from integer without a cast [-Wint-conversion]
({ \
^
include/linux/percpu-defs.h:494:29: note: in expansion of macro '__pcpu_size_call_return'
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
^~~~~~~~~~~~~~~~~~~~~~~
>> kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~ At top level:
kernel/fork.c:173:12: warning: 'free_vm_stack_cache' defined but not used [-Wunused-function]
static int free_vm_stack_cache(unsigned int cpu)
^~~~~~~~~~~~~~~~~~~
vim +/NR_CACHED_STACKS +177 kernel/fork.c
8 * 'fork.c' contains the help-routines for the 'fork' system call
9 * (see also entry.S and others).
10 * Fork is rather simple, once you get the hang of it, but the memory
11 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
12 */
13
> 14 #include <linux/slab.h>
15 #include <linux/init.h>
16 #include <linux/unistd.h>
17 #include <linux/module.h>
18 #include <linux/vmalloc.h>
19 #include <linux/completion.h>
20 #include <linux/personality.h>
21 #include <linux/mempolicy.h>
22 #include <linux/sem.h>
23 #include <linux/file.h>
24 #include <linux/fdtable.h>
25 #include <linux/iocontext.h>
26 #include <linux/key.h>
27 #include <linux/binfmts.h>
28 #include <linux/mman.h>
29 #include <linux/mmu_notifier.h>
30 #include <linux/fs.h>
31 #include <linux/mm.h>
32 #include <linux/vmacache.h>
33 #include <linux/nsproxy.h>
34 #include <linux/capability.h>
35 #include <linux/cpu.h>
36 #include <linux/cgroup.h>
37 #include <linux/security.h>
38 #include <linux/hugetlb.h>
39 #include <linux/seccomp.h>
40 #include <linux/swap.h>
41 #include <linux/syscalls.h>
42 #include <linux/jiffies.h>
43 #include <linux/futex.h>
44 #include <linux/compat.h>
45 #include <linux/kthread.h>
46 #include <linux/task_io_accounting_ops.h>
47 #include <linux/rcupdate.h>
48 #include <linux/ptrace.h>
49 #include <linux/mount.h>
50 #include <linux/audit.h>
51 #include <linux/memcontrol.h>
52 #include <linux/ftrace.h>
53 #include <linux/proc_fs.h>
54 #include <linux/profile.h>
55 #include <linux/rmap.h>
56 #include <linux/ksm.h>
57 #include <linux/acct.h>
58 #include <linux/userfaultfd_k.h>
59 #include <linux/tsacct_kern.h>
60 #include <linux/cn_proc.h>
61 #include <linux/freezer.h>
62 #include <linux/delayacct.h>
63 #include <linux/taskstats_kern.h>
64 #include <linux/random.h>
65 #include <linux/tty.h>
66 #include <linux/blkdev.h>
67 #include <linux/fs_struct.h>
68 #include <linux/magic.h>
69 #include <linux/perf_event.h>
70 #include <linux/posix-timers.h>
71 #include <linux/user-return-notifier.h>
72 #include <linux/oom.h>
73 #include <linux/khugepaged.h>
74 #include <linux/signalfd.h>
75 #include <linux/uprobes.h>
76 #include <linux/aio.h>
77 #include <linux/compiler.h>
78 #include <linux/sysctl.h>
79 #include <linux/kcov.h>
80
81 #include <asm/pgtable.h>
82 #include <asm/pgalloc.h>
83 #include <linux/uaccess.h>
84 #include <asm/mmu_context.h>
85 #include <asm/cacheflush.h>
86 #include <asm/tlbflush.h>
87
88 #include <trace/events/sched.h>
89
90 #define CREATE_TRACE_POINTS
91 #include <trace/events/task.h>
92
93 /*
94 * Minimum number of threads to boot the kernel
95 */
96 #define MIN_THREADS 20
97
98 /*
99 * Maximum number of threads
100 */
101 #define MAX_THREADS FUTEX_TID_MASK
102
103 /*
104 * Protected counters by write_lock_irq(&tasklist_lock)
105 */
106 unsigned long total_forks; /* Handle normal Linux uptimes. */
107 int nr_threads; /* The idle threads do not count.. */
108
109 int max_threads; /* tunable limit on nr_threads */
110
111 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
112
113 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
114
115 #ifdef CONFIG_PROVE_RCU
116 int lockdep_tasklist_lock_is_held(void)
117 {
118 return lockdep_is_held(&tasklist_lock);
119 }
120 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
121 #endif /* #ifdef CONFIG_PROVE_RCU */
122
123 int nr_processes(void)
124 {
125 int cpu;
126 int total = 0;
127
128 for_each_possible_cpu(cpu)
129 total += per_cpu(process_counts, cpu);
130
131 return total;
132 }
133
134 void __weak arch_release_task_struct(struct task_struct *tsk)
135 {
136 }
137
138 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
139 static struct kmem_cache *task_struct_cachep;
140
141 static inline struct task_struct *alloc_task_struct_node(int node)
142 {
143 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
144 }
145
146 static inline void free_task_struct(struct task_struct *tsk)
147 {
148 kmem_cache_free(task_struct_cachep, tsk);
149 }
150 #endif
151
152 void __weak arch_release_thread_stack(unsigned long *stack)
153 {
154 }
155
156 #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
157
158 /*
159 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
160 * kmemcache based allocator.
161 */
162 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
163
164 #ifdef CONFIG_VMAP_STACK
165 /*
166 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
167 * flush. Try to minimize the number of calls by caching stacks.
168 */
169 #define NR_CACHED_STACKS 2
170 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
171 #endif
172
173 static int free_vm_stack_cache(unsigned int cpu)
174 {
175 int i;
176
> 177 for (i = 0; i < NR_CACHED_STACKS; i++) {
> 178 struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]); 179 if (!vm_stack)
180 continue;
181
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143 config: i386-randconfig-x004-201706 (attached as .config) compiler…
Hi Hoeun,
[auto build test WARNING on next-20170208]
[also build test WARNING on v4.10-rc7]
[cannot apply to linus/master linux/master v4.9-rc8 v4.9-rc7 v4.9-rc6]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143
config: i386-randconfig-x004-201706 (attached as .config)
compiler: gcc-6 (Debian 6.2.0-3) 6.2.0 20160901 reproduce:
# save the attached .config to linux build tree
make ARCH=i386 All warnings (new ones prefixed by >>):
kernel/fork.c: In function 'free_vm_stack_cache':
kernel/fork.c:177:18: error: 'NR_CACHED_STACKS' undeclared (first use in this function)
for (i = 0; i < NR_CACHED_STACKS; i++) {
^~~~~~~~~~~~~~~~
kernel/fork.c:177:18: note: each undeclared identifier is reported only once for each function it appears in
In file included from include/asm-generic/percpu.h:6:0,
from arch/x86/include/asm/percpu.h:542,
from arch/x86/include/asm/preempt.h:5,
from include/linux/preempt.h:59,
from include/linux/spinlock.h:50,
from include/linux/mmzone.h:7,
from include/linux/gfp.h:5,
from include/linux/slab.h:14,
from kernel/fork.c:14:
kernel/fork.c:178:46: error: 'cached_stacks' undeclared (first use in this function)
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^
include/linux/percpu-defs.h:305:9: note: in definition of macro '__pcpu_size_call_return'
typeof(variable) pscr_ret__; \
^~~~~~~~
kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~
include/linux/percpu-defs.h:304:1: warning: initialization makes pointer from integer without a cast [-Wint-conversion]
({ \
^
include/linux/percpu-defs.h:494:29: note: in expansion of macro '__pcpu_size_call_return'
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
^~~~~~~~~~~~~~~~~~~~~~~
kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~
In file included from arch/x86/include/asm/preempt.h:5:0,
from include/linux/preempt.h:59,
from include/linux/spinlock.h:50,
from include/linux/mmzone.h:7,
from include/linux/gfp.h:5,
from include/linux/slab.h:14,
from kernel/fork.c:14:
>> arch/x86/include/asm/percpu.h:94:13: warning: assignment makes integer from pointer without a cast [-Wint-conversion]
pto_tmp__ = (val); \
^
arch/x86/include/asm/percpu.h:416:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:364:11: note: in expansion of macro 'this_cpu_write_1'
case 1: stem##1(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
>> kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~
>> arch/x86/include/asm/percpu.h:94:13: warning: assignment makes integer from pointer without a cast [-Wint-conversion]
pto_tmp__ = (val); \
^
arch/x86/include/asm/percpu.h:417:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:365:11: note: in expansion of macro 'this_cpu_write_2'
case 2: stem##2(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
>> kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~
>> arch/x86/include/asm/percpu.h:94:13: warning: assignment makes integer from pointer without a cast [-Wint-conversion]
pto_tmp__ = (val); \
^
arch/x86/include/asm/percpu.h:418:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:366:11: note: in expansion of macro 'this_cpu_write_4'
case 4: stem##4(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
>> kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~ At top level:
kernel/fork.c:173:12: warning: 'free_vm_stack_cache' defined but not used [-Wunused-function]
static int free_vm_stack_cache(unsigned int cpu)
^~~~~~~~~~~~~~~~~~~
vim +/this_cpu_write +183 kernel/fork.c
167 * flush. Try to minimize the number of calls by caching stacks.
168 */
169 #define NR_CACHED_STACKS 2
170 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
171 #endif
172
173 static int free_vm_stack_cache(unsigned int cpu)
174 {
175 int i;
176
177 for (i = 0; i < NR_CACHED_STACKS; i++) {
178 struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
179 if (!vm_stack)
180 continue;
181
182 vfree(vm_stack->addr);
> 183 this_cpu_write(cached_stacks[i], NULL);
184 }
185
186 return 0;
187 }
188
189 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
190 {
191 #ifdef CONFIG_VMAP_STACK
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
Quoted 20 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the cached stacks may remain unfreed even when cpu are offline.
By adding a cpu hotplug callback to free the cached stacks when a cpu
goes offline, the pages of the cached stacks are not wasted.Signed-off-by Hoeun Ryu <[email protected]>---
Changes in v2:
remove cpuhp callback forstarup, onlyteardowncallback is installed.kernel/fork.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)kernel/fork.c +0
-0
kernel/fork.c void __weak arch_release_thread_stack(unsigned long *stack) · line 178
Quoted 9 lines
kernel/fork.c +6
-0@@ -170,6 +170,22 @@ void __weak arch_release_thread_stack(unsigned long *stack) static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif +static int free_vm_stack_cache (unsigned int cpu) +{ +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack =this_cpu_read (cached_stacks[i]);
the callbak will run on the given cpu so this_cpu_read will be in fact per_cpu_ptr(cached_stacks[i], cpu). Using this_cpu_read is just too confusing. Also you do want to make this function defined only for CONFIG_VMAP_STACK.
Quoted 10 lines
kernel/fork.c +10
-0+if (!vm_stack) +continue ; + +vfree (vm_stack->addr); +this_cpu_write (cached_stacks[i],NULL ); + } + +return 0 ; +} +
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 24 lines · Michal Hocko
Quoted 23 lines · Hoeun Ryu
Using virtually mapped stack, kernel stacks are allocated via vmalloc.
In the current implementation, two stacks per cpu can be cached when tasks are freed and the cached stacks are used again in task duplications. but the cached stacks may remain unfreed even when cpu are offline.
By adding a cpu hotplug callback to free the cached stacks when a cpu
goes offline, the pages of the cached stacks are not wasted.Signed-off-by Hoeun Ryu <[email protected]>---
Changes in v2:
remove cpuhp callback forstarup, onlyteardowncallback is installed.kernel/fork.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+)kernel/fork.c +0
-0@@ -170,6 +170,22 @@ void __weak arch_release_thread_stack(unsigned long *stack) static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);#endif
Quoted 12 lines
kernel/fork.c void __weak arch_release_thread_stack(unsigned long *stack) · line 178
Quoted 6 lines
kernel/fork.c +6
-0+static int free_vm_stack_cache (unsigned int cpu) +{ +int i; + +for (i =0 ; i <NR_CACHED_STACKS ; i++) { +struct vm_struct *vm_stack =this_cpu_read (cached_stacks[i]);the callbak will run on the given cpu so
this_cpu_readwill be in factper_cpu_ptr(cached_stacks[i], cpu). Usingthis_cpu_readis just too confusing. Also you do want to make this function defined only forCONFIG_VMAP_STACK.
Sorry, Thank you for your correction. I will fix this.
Quoted 14 lines
kernel/fork.c void __weak arch_release_thread_stack(unsigned long *stack) · line 188
Quoted 10 lines
kernel/fork.c +10
-0+if (!vm_stack) +continue ; + +vfree (vm_stack->addr); +this_cpu_write (cached_stacks[i],NULL ); + } + +return 0 ; +} +--
Michal Hocko
SUSE Labs
kbuild test robot
build boturl: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143 reproduce:
Hi Hoeun,
[auto build test WARNING on next-20170208]
[also build test WARNING on v4.10-rc7]
[cannot apply to linus/master linux/master v4.9-rc8 v4.9-rc7 v4.9-rc6]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Hoeun-Ryu/fork-free-vmapped-stacks-in-cache-when-cpus-are-offline/20170209-124143
reproduce:
# apt-get install sparse
make ARCH=x86_64 allmodconfig
make C=1 CF=-D__CHECK_ENDIAN__sparse warnings: (new ones prefixed by >>)
include/linux/compiler.h:264:8: sparse: attribute 'no_sanitize_address': unknown attribute kernel/fork.c:177:25: sparse: undefined identifier 'NR_CACHED_STACKS' kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks' kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:178:46: sparse: undefined identifier 'cached_stacks'
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for operation (+)
kernel/fork.c:178:46: left side has type bad type
kernel/fork.c:178:46: right side has type int [signed] [assigned] i
>> kernel/fork.c:178:46: sparse: cannot dereference this type
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
>> kernel/fork.c:178:46: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for operation (+)
kernel/fork.c:183:17: left side has type bad type
kernel/fork.c:183:17: right side has type int [signed] [assigned] i
kernel/fork.c:183:17: sparse: cannot dereference this type
kernel/fork.c:183:17: sparse: incompatible types for operation (+)
kernel/fork.c:183:17: left side has type bad type
kernel/fork.c:183:17: right side has type int
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for operation (+)
kernel/fork.c:183:17: left side has type bad type
kernel/fork.c:183:17: right side has type int [signed] [assigned] i
kernel/fork.c:183:17: sparse: cannot dereference this type
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for operation (+)
kernel/fork.c:183:17: left side has type bad type
kernel/fork.c:183:17: right side has type int [signed] [assigned] i
kernel/fork.c:183:17: sparse: cannot dereference this type
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for operation (+)
kernel/fork.c:183:17: left side has type bad type
kernel/fork.c:183:17: right side has type int [signed] [assigned] i
kernel/fork.c:183:17: sparse: cannot dereference this type
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: undefined identifier 'cached_stacks'
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
kernel/fork.c:183:17: sparse: incompatible types for 'case' statement
>> kernel/fork.c:183:17: sparse: too many errors
kernel/fork.c: In function 'free_vm_stack_cache':
kernel/fork.c:177:18: error: 'NR_CACHED_STACKS' undeclared (first use in this function)
for (i = 0; i < NR_CACHED_STACKS; i++) {
^~~~~~~~~~~~~~~~
kernel/fork.c:177:18: note: each undeclared identifier is reported only once for each function it appears in
In file included from include/asm-generic/percpu.h:6:0,
from arch/x86/include/asm/percpu.h:542,
from arch/x86/include/asm/preempt.h:5,
from include/linux/preempt.h:59,
from include/linux/spinlock.h:50,
from include/linux/mmzone.h:7,
from include/linux/gfp.h:5,
from include/linux/slab.h:14,
from kernel/fork.c:14:
kernel/fork.c:178:46: error: 'cached_stacks' undeclared (first use in this function)
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^
include/linux/percpu-defs.h:305:9: note: in definition of macro '__pcpu_size_call_return'
typeof(variable) pscr_ret__; \
^~~~~~~~
kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~
include/linux/percpu-defs.h:304:1: warning: initialization makes pointer from integer without a cast [-Wint-conversion]
({ \
^
include/linux/percpu-defs.h:494:29: note: in expansion of macro '__pcpu_size_call_return'
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
^~~~~~~~~~~~~~~~~~~~~~~
kernel/fork.c:178:32: note: in expansion of macro 'this_cpu_read'
struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
^~~~~~~~~~~~~
In file included from arch/x86/include/asm/preempt.h:5:0,
from include/linux/preempt.h:59,
from include/linux/spinlock.h:50,
from include/linux/mmzone.h:7,
from include/linux/gfp.h:5,
from include/linux/slab.h:14,
from kernel/fork.c:14:
arch/x86/include/asm/percpu.h:94:13: warning: assignment makes integer from pointer without a cast [-Wint-conversion]
pto_tmp__ = (val); \
^
arch/x86/include/asm/percpu.h:416:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:364:11: note: in expansion of macro 'this_cpu_write_1'
case 1: stem##1(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~
arch/x86/include/asm/percpu.h:101:15: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
: "qi" ((pto_T__)(val))); \
^
arch/x86/include/asm/percpu.h:416:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:364:11: note: in expansion of macro 'this_cpu_write_1'
case 1: stem##1(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~
arch/x86/include/asm/percpu.h:106:15: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
: "ri" ((pto_T__)(val))); \
^
arch/x86/include/asm/percpu.h:416:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:364:11: note: in expansion of macro 'this_cpu_write_1'
case 1: stem##1(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~
arch/x86/include/asm/percpu.h:111:15: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
: "ri" ((pto_T__)(val))); \
^
arch/x86/include/asm/percpu.h:416:36: note: in expansion of macro 'percpu_to_op'
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
^~~~~~~~~~~~
include/linux/percpu-defs.h:364:11: note: in expansion of macro 'this_cpu_write_1'
case 1: stem##1(variable, __VA_ARGS__);break; \
^~~~
include/linux/percpu-defs.h:495:34: note: in expansion of macro '__pcpu_size_call'
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
^~~~~~~~~~~~~~~~
kernel/fork.c:183:3: note: in expansion of macro 'this_cpu_write'
this_cpu_write(cached_stacks[i], NULL);
^~~~~~~~~~~~~~ arch/x86/include/asm/percpu.h:116:15: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
vim +178 kernel/fork.c
171 #endif
172
173 static int free_vm_stack_cache(unsigned int cpu)
174 {
175 int i;
176
> 177 for (i = 0; i < NR_CACHED_STACKS; i++) {
> 178 struct vm_struct *vm_stack = this_cpu_read(cached_stacks[i]);
179 if (!vm_stack)
180 continue;
181
182 vfree(vm_stack->addr);
> 183 this_cpu_write(cached_stacks[i], NULL); 184 }
185
186 return 0;
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation