mm: don't call should_failslab() for !CONFIG_FAILSLAB
Subsystems (3)
PATCH 1/1 mm: don't call should_failslab() for !CONFIG_FAILSLAB
Allocations can be a very hot path, and this out-of-line function call is noticeable.
---
include/linux/fault-inject.h +1 -1
@@ -64,8 +64,8 @@ static inline struct dentry *fault_create_debugfs_attr(const char *name,struct kmem_cache ; -int should_failslab (struct kmem_cache *s,gfp_t gfpflags);#ifdef CONFIG_FAILSLAB +int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags)
mm/slab.h +2 -0
@@ -491,8 +491,10 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,might_alloc (flags); +#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; +#endif if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;
mm/slab_common.c +2 -0
@@ -1323,6 +1323,7 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);EXPORT_TRACEPOINT_SYMBOL (kfree);EXPORT_TRACEPOINT_SYMBOL (kmem_cache_free); +#ifdef CONFIG_FAILSLAB int should_failslab (struct kmem_cache *s,gfp_t gfpflags) {if (__should_failslab (s, gfpflags))
@@ -1330,3 +1331,4 @@ int should_failslab(struct kmem_cache *s, gfp_t gfpflags) return 0; }ALLOW_ERROR_INJECTION (should_failslab,ERRNO ); +#endif
Signature · 1 line
Jens Axboe
Andrew Morton
maintainerQuoted 20 lines · On Tue, 5 Oct 2021 09:31:43 -0600 Jens Axboe <[email protected]> wrote:
Allocations can be a very hot path, and this out-of-line function call is noticeable.
include/linux/fault-inject.h +1
-1@@ -64,8 +64,8 @@ static inline struct dentry *fault_create_debugfs_attr(const char *name,struct kmem_cache ; -int should_failslab (struct kmem_cache *s,gfp_t gfpflags);#ifdef CONFIG_FAILSLAB +int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags)mm/slab.h +0
-0
mm/slab.h static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, · line 497
Quoted 7 lines
mm/slab.h +2
-0@@ -491,8 +491,10 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,might_alloc (flags); +#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; +#endif
Can we avoid the ifdefs here?
Quoted 16 lines
mm/slab.h +0
-0if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;mm/slab_common.c +1
-0@@ -1323,6 +1323,7 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);EXPORT_TRACEPOINT_SYMBOL (kfree);EXPORT_TRACEPOINT_SYMBOL (kmem_cache_free); +#ifdef CONFIG_FAILSLAB int should_failslab (struct kmem_cache *s,gfp_t gfpflags) {if (__should_failslab (s, gfpflags))
mm/slab_common.c int should_failslab(struct kmem_cache *s, gfp_t gfpflags) · line 1334
Quoted 4 lines
mm/slab_common.c +1
-0@@ -1330,3 +1331,4 @@ int should_failslab(struct kmem_cache *s, gfp_t gfpflags) return 0; }ALLOW_ERROR_INJECTION (should_failslab,ERRNO ); +#endif
Like,
include/linux/fault-inject.h renamed from include/linux/fault-inject.h~mm-dont-call-should_failslab-for-config_failslab-fix +4 -0
@@ -68,6 +68,10 @@ struct kmem_cache;int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else +static inline int should_failslab (struct kmem_cache *s,gfp_t gfpflags) +{ +return 0 ; +}static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags) {return false;
mm/slab.h renamed from mm/slab.h~mm-dont-call-should_failslab-for-config_failslab-fix +0 -2
@@ -491,10 +491,8 @@ static inline struct kmem_cache *slab_prmight_alloc (flags); -#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; -#endif if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;
_
Quoted 81 lines · Andrew Morton
Quoted 20 lines · On Tue, 5 Oct 2021 09:31:43 -0600 Jens Axboe <[email protected]> wrote:
Allocations can be a very hot path, and this out-of-line function call is noticeable.
include/linux/fault-inject.h +1
-1@@ -64,8 +64,8 @@ static inline struct dentry *fault_create_debugfs_attr(const char *name,struct kmem_cache ; -int should_failslab (struct kmem_cache *s,gfp_t gfpflags);#ifdef CONFIG_FAILSLAB +int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags)mm/slab.h +0
-0mm/slab.h static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, · line 497
Quoted 7 lines
mm/slab.h +2
-0@@ -491,8 +491,10 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,might_alloc (flags); +#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; +#endif Can we avoid the ifdefs here?
Quoted 16 lines
mm/slab.h +0
-0if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;mm/slab_common.c +1
-0@@ -1323,6 +1323,7 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);EXPORT_TRACEPOINT_SYMBOL (kfree);EXPORT_TRACEPOINT_SYMBOL (kmem_cache_free); +#ifdef CONFIG_FAILSLAB int should_failslab (struct kmem_cache *s,gfp_t gfpflags) {if (__should_failslab (s, gfpflags))mm/slab_common.c int should_failslab(struct kmem_cache *s, gfp_t gfpflags) · line 1334
Quoted 4 lines
mm/slab_common.c +1
-0@@ -1330,3 +1331,4 @@ int should_failslab(struct kmem_cache *s, gfp_t gfpflags) return 0; }ALLOW_ERROR_INJECTION (should_failslab,ERRNO ); +#endif Like,
include/linux/fault-inject.h renamed from include/linux/fault-inject.h~mm-dont-call-should_failslab-for-config_failslab-fix +4
-0@@ -68,6 +68,10 @@ struct kmem_cache;int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else +static inline int should_failslab (struct kmem_cache *s,gfp_t gfpflags) +{ +return 0 ; +}static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags) {return false;mm/slab.h renamed from mm/slab.h~mm-dont-call-should_failslab-for-config_failslab-fix +0
-2@@ -491,10 +491,8 @@ static inline struct kmem_cache *slab_prmight_alloc (flags); -#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; -#endif if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;
Quoted 1 line
_
Yep, that'll work!
Signature · 1 line
Jens Axboe
Quoted 4 lines · Jens Axboe
Allocations can be a very hot path, and this out-of-line function call is noticeable.
Signed-off-by Jens Axboe <[email protected]>
It used to be inline b4 (hi, Konstantin!) and then was converted to be like this intentionally :/
See 4f6923fbb352 ("mm: make should_failslab always available for fault injection")
And now also kernel/bpf/verifier.c contains:BTF_ID(func, should_failslab)
I think either your or Andrew's version will break this BTF_ID thing, at the very least.
But I do strongly agree that putting unconditionally a non-inline call into slab allocator fastpath sucks. Can we make it so that bpf can only do these overrides when CONFIG_FAILSLAB is enabled?
I don't know, perhaps putting this BTF_ID() in #ifdef as well, or providing a dummy that is always available (so that nothing breaks), but doesn't actually affect slab_pre_alloc_hook() unless CONFIG_FAILSLAB has been enabled?
Quoted 49 lines
---
include/linux/fault-inject.h +1
-1@@ -64,8 +64,8 @@ static inline struct dentry *fault_create_debugfs_attr(const char *name,struct kmem_cache ; -int should_failslab (struct kmem_cache *s,gfp_t gfpflags);#ifdef CONFIG_FAILSLAB +int should_failslab (struct kmem_cache *s,gfp_t gfpflags);extern bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags);#else static inline bool __should_failslab (struct kmem_cache *s,gfp_t gfpflags)mm/slab.h +2
-0@@ -491,8 +491,10 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,might_alloc (flags); +#ifdef CONFIG_FAILSLAB if (should_failslab (s, flags))return NULL ; +#endif if (!memcg_slab_pre_alloc_hook (s, objcgp, size, flags))return NULL ;mm/slab_common.c +2
-0@@ -1323,6 +1323,7 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);EXPORT_TRACEPOINT_SYMBOL (kfree);EXPORT_TRACEPOINT_SYMBOL (kmem_cache_free); +#ifdef CONFIG_FAILSLAB int should_failslab (struct kmem_cache *s,gfp_t gfpflags) {if (__should_failslab (s, gfpflags))@@ -1330,3 +1331,4 @@ int should_failslab(struct kmem_cache *s, gfp_t gfpflags) return 0; }ALLOW_ERROR_INJECTION (should_failslab,ERRNO ); +#endif
Quoted 18 lines · Vlastimil Babka
Quoted 4 lines · Jens Axboe
Allocations can be a very hot path, and this out-of-line function call is noticeable.
Signed-off-by Jens Axboe <[email protected]>It used to be inline b4 (hi, Konstantin!) and then was converted to be like this intentionally :/
See
4f6923fbb352("mm: makeshould_failslabalways available for fault injection")And now also
kernel/bpf/verifier.ccontains:BTF_ID(func, should_failslab)I think either your or Andrew's version will break this
BTF_IDthing, at the very least.
Quoted 6 lines
But I do strongly agree that putting unconditionally a non-inline call into slab allocator fastpath sucks. Can we make it so that bpf can only do these overrides when
CONFIG_FAILSLABis enabled?
I don't know, perhaps putting thisBTF_ID()in #ifdef as well, or providing a dummy that is always available (so that nothing breaks), but doesn't actually affectslab_pre_alloc_hook()unlessCONFIG_FAILSLABhas been enabled?
That seems to be the right approach, limiting it on it actually being enabled and a function call.
Signature · 1 line
Jens Axboe
Quoted 7 lines · Vlastimil Babka
Quoted 4 lines · Jens Axboe
Allocations can be a very hot path, and this out-of-line function call is noticeable.
Signed-off-by Jens Axboe <[email protected]>It used to be inline b4 (hi, Konstantin!)
Congratulations, you made me look. :)
-K
+cc Linus
Quoted 18 lines · Vlastimil Babka
Quoted 4 lines · Jens Axboe
Allocations can be a very hot path, and this out-of-line function call is noticeable.
Signed-off-by Jens Axboe <[email protected]>It used to be inline b4 (hi, Konstantin!) and then was converted to be like this intentionally :/
See
4f6923fbb352("mm: makeshould_failslabalways available for fault injection")And now also
kernel/bpf/verifier.ccontains:BTF_ID(func, should_failslab)I think either your or Andrew's version will break this
BTF_IDthing, at the very least.
Quoted 7 lines
But I do strongly agree that putting unconditionally a non-inline call into slab allocator fastpath sucks. Can we make it so that bpf can only do these overrides when
CONFIG_FAILSLABis enabled?
I don't know, perhaps putting thisBTF_ID()in #ifdef as well, or providing a dummy that is always available (so that nothing breaks), but doesn't actually affectslab_pre_alloc_hook()unlessCONFIG_FAILSLABhas been enabled?
I just ran into it while looking at kmalloc + kfree pair.
A toy test which calls this in a loop like so:
static long noinline custom_bench(void)
{
void *buf;
while (!signal_pending(current)) {
buf = kmalloc(16, GFP_KERNEL);
kfree(buf);
cond_resched();
}
return -EINTR;
}... shows this with perf top:
57.88% [kernel] [k] kfree
31.38% [kernel] [k] kmalloc_trace_noprof
3.20% [kernel] [k] should_failslab.constprop.0A side note is that I verified majority of the time in kfree and kmalloc_trace_noprof is cmpxchg16b, which is both good and bad news.
As for should_failslab, it compiles to an empty func on production kernels and is present even when there are no supported means of instrumenting it. As in everyone pays for its existence, even if there is no way to use it.
Also note there are 3 unrelated mechanisms to alter the return code, which imo is 2 too many. But more importantly they are not even coordinated.
A hard requirement for a long term solution is to not alter the fast path beyond nops for hot patching.
So far I think implementing this in a clean manner would require agreeing on some namespace for bpf ("failprobes"?) and coordinating hotpatching between different mechanisms. Maybe there is a better, I don't know.
Here is the crux of my e-mail though:
1. turning should_failslab into a mandatory func call is an ok local hack for the test farm, not a viable approach for production
2. as such it is up to the original submitter (or whoever else who wants to pick up the slack) to implement something which hotpatches the callsite as opposed to inducing a function call for everyone
In the meantime the routine should disappear unless explicitly included in kernel config. The patch submitted here would be one way to do it.
Quoted 62 lines · Mateusz Guzik
+cc Linus
Quoted 18 lines · Vlastimil Babka
Quoted 4 lines · Jens Axboe
Allocations can be a very hot path, and this out-of-line function call is noticeable.
Signed-off-by Jens Axboe <[email protected]>It used to be inline b4 (hi, Konstantin!) and then was converted to be like this intentionally :/
See
4f6923fbb352("mm: makeshould_failslabalways available for fault injection")And now also
kernel/bpf/verifier.ccontains:BTF_ID(func, should_failslab)I think either your or Andrew's version will break this
BTF_IDthing, at the very least.Quoted 7 lines
But I do strongly agree that putting unconditionally a non-inline call into slab allocator fastpath sucks. Can we make it so that bpf can only do these overrides when
CONFIG_FAILSLABis enabled?
I don't know, perhaps putting thisBTF_ID()in #ifdef as well, or providing a dummy that is always available (so that nothing breaks), but doesn't actually affectslab_pre_alloc_hook()unlessCONFIG_FAILSLABhas been enabled?I just ran into it while looking at kmalloc + kfree pair.
A toy test which calls this in a loop like so:
static long noinlinecustom_bench(void){ void *buf; while (!signal_pending(current)) { buf = kmalloc(16, GFP_KERNEL); kfree(buf); cond_resched(); } return -EINTR; }... shows this with perf top:
57.88% [kernel] [k] kfree 31.38% [kernel] [k] kmalloc_trace_noprof 3.20% [kernel] [k] should_failslab.constprop.0A side note is that I verified majority of the time in kfree and
kmalloc_trace_noprofis cmpxchg16b, which is both good and bad news.As for
should_failslab, it compiles to an empty func on production kernels and is present even when there are no supported means of instrumenting it. As in everyone pays for its existence, even if there is no way to use it.Also note there are 3 unrelated mechanisms to alter the return code, which imo is 2 too many. But more importantly they are not even coordinated.
Quoted 7 lines
A hard requirement for a long term solution is to not alter the fast path beyond nops for hot patching.
So far I think implementing this in a clean manner would require agreeing on some namespace for bpf ("failprobes"?) and coordinating hotpatching between different mechanisms. Maybe there is a better, I don't know.
I've attempted something (not complete yet) here:
https://lore.kernel.org/all/[email protected]/
Quoted 10 lines
Here is the crux of my e-mail though:
1. turningshould_failslabinto a mandatory func call is an ok local hack for the test farm, not a viable approach for production
2. as such it is up to the original submitter (or whoever else who wants to pick up the slack) to implement something which hotpatches the callsite as opposed to inducing a function call for everyoneIn the meantime the routine should disappear unless explicitly included in kernel config. The patch submitted here would be one way to do it.