common kmalloc subsystem on SLAB/SLUB
Subsystems (3)
cover common kmalloc subsystem on SLAB/SLUB
Hello, this series is cleanup of slab common code.
After this series, kmalloc subsystem is perfectly generalized between SLAB and SLUB.
This series is not small, and some of review and discussion may be needed. But I bet you will like this! :D
Many thanks to Matthew, Marco, Vlastimil who gave comments in previous series.
Any feedbacks will be appreciated.
Thanks!
========= series description ==========
patch 1 makes slab_alloc_node() in SLAB available for non-NUMA configurations for further cleanup.
patch 2-8 are cleanup of unnecessary CONFIG_TRACING/NUMA ifdefs and duplicate code. (and small optimization of kmalloc_node())
patch 9 makes SLAB pass requests larger than order-1 page to page allocator. This is useful for further generalization.
patch 10-11 are cleanup of tracepoints. Currently there are five trace points in slab: kmalloc, kmalloc_node, kmem_cache_alloc, kmem_cahce_alloc_node, kmem_cache_free, kfree.
patch 10-11 make tracepoints print cache name and convert every tracepoint to kmem_cache_alloc_node and kmem_cache_free.
patch 12 generalizes whole kmalloc subsystem on SLAB and SLUB.
patch 13 removes kmem_cache_alloc_node() that became useless after patch 12.
patch 14-15 are small improvements of __ksize().
Hyeonggon Yoo (15):
mm/slab: cleanup slab_alloc() and slab_alloc_node() mm/sl[auo]b: remove CONFIG_NUMA ifdefs for common functions mm/sl[au]b: remove CONFIG_TRACING ifdefs for tracing functions mm/sl[auo]b: fold kmalloc_order() into kmalloc_large() mm/slub: move kmalloc_large_node() to slab_common.c mm/slab_common: cleanup kmalloc_large()
mm/sl[au]b: kmalloc_node: pass large requests to page allocator mm/sl[auo]b: cleanup kmalloc()
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[auo]b: print cache name in tracepoints mm/sl[auo]b: use same tracepoint in kmalloc and normal caches mm/sl[au]b: generalize kmalloc subsystem
mm/sl[au]b: remove kmem_cache_alloc_node_trace() mm/sl[auo]b: move definition of __ksize() to mm/slab.h mm/sl[au]b: check if large object is valid in __ksize()
include/linux/slab.h | 252 +++++++++++--------------- include/trace/events/kmem.h | 107 ++--------- mm/slab.c | 347 ++++++++---------------------------- mm/slab.h | 9 + mm/slab_common.c | 128 ++++++++++--- mm/slob.c | 69 +++---- mm/slub.c | 237 ++---------------------- 7 files changed, 346 insertions(+), 803 deletions(-)
Signature · 1 line
2.33.1
RFC PATCH 1/15 mm/slab: cleanup slab_alloc() and slab_alloc_node()
Make slab_alloc_node() available for non-NUMA configurations and make slab_alloc() wrapper of slab_alloc_node(). This is necessary for further cleanup.
Do not check availability of node when allocating from locally cached objects. It's redundant.
This patch was tested on both CONFIG_NUMA=y and n.
--- mm/slab.c | 116 +++++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 66 deletions(-)
mm/slab.c +50 -66
@@ -3200,60 +3200,6 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,return obj ? obj :fallback_alloc (cachep, flags); } -static __always_inline void * -slab_alloc_node (struct kmem_cache *cachep,gfp_t flags,int nodeid,size_t orig_size, -unsigned long caller) -{ -unsigned long save_flags; -void *ptr; -int slab_node =numa_mem_id (); -struct obj_cgroup *objcg =NULL ; -bool init = false; - - flags &= gfp_allowed_mask; - cachep =slab_pre_alloc_hook (cachep, &objcg,1 , flags); -if (unlikely (!cachep)) -return NULL ; - - ptr =kfence_alloc (cachep, orig_size, flags); -if (unlikely (ptr)) - goto out_hooks; - -cache_alloc_debugcheck_before (cachep, flags); -local_irq_save (save_flags); - -if (nodeid ==NUMA_NO_NODE ) - nodeid = slab_node; - -if (unlikely (!get_node (cachep, nodeid))) { -/* Node not bootstrapped yet */ - ptr =fallback_alloc (cachep, flags); - goto out; - } - -if (nodeid == slab_node) { -/* -* Use the locally cached objects if possible. -* However ____cache_alloc does not allow fallback -* to other nodes. It may fail while we still have -* objects on other nodes available. -*/ - ptr =____cache_alloc (cachep, flags); -if (ptr) - goto out; - } -/* ___cache_alloc_node can fall back to other nodes */ - ptr =____cache_alloc_node (cachep, flags, nodeid); - out: -local_irq_restore (save_flags); - ptr =cache_alloc_debugcheck_after (cachep, flags, ptr, caller); - init =slab_want_init_on_alloc (flags, cachep); - -out_hooks: -slab_post_alloc_hook (cachep, objcg, flags,1 , &ptr, init); -return ptr; -} - static __always_inline void *__do_cache_alloc (struct kmem_cache *cache,gfp_t flags) {
@@ -3283,14 +3229,24 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) {return ____cache_alloc (cachep, flags); } -#endif /* CONFIG_NUMA */ +static __always_inline bool node_match(int nodeid, int slab_node) +{ +#ifdef CONFIG_NUMA +if (nodeid !=NUMA_NO_NODE && nodeid != slab_node) +return false; +#endif +return true; +} + static __always_inline void * -slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +slab_alloc_node (struct kmem_cache *cachep,gfp_t flags,int nodeid,size_t orig_size, +unsigned long caller) {unsigned long save_flags; -void *objp; +void *ptr; +int slab_node =numa_mem_id ();struct obj_cgroup *objcg =NULL ;bool init = false;
@@ -3299,21 +3255,49 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned loif (unlikely (!cachep))return NULL ; - objp =kfence_alloc (cachep, orig_size, flags); -if (unlikely (objp)) - goto out; + ptr =kfence_alloc (cachep, orig_size, flags); +if (unlikely (ptr)) + goto out_hooks;cache_alloc_debugcheck_before (cachep, flags);local_irq_save (save_flags); - objp =__do_cache_alloc (cachep, flags); + +if (node_match (nodeid, slab_node)) { +/* +* Use the locally cached objects if possible. +* However ____cache_alloc does not allow fallback +* to other nodes. It may fail while we still have +* objects on other nodes available. +*/ + ptr =____cache_alloc (cachep, flags); +if (ptr) + goto out; + } +#ifdef CONFIG_NUMA + else if (unlikely(!get_node(cachep, nodeid))) { +/* Node not bootstrapped yet */ + ptr =fallback_alloc (cachep, flags); + goto out; + } + +/* ___cache_alloc_node can fall back to other nodes */ + ptr =____cache_alloc_node (cachep, flags, nodeid); +#endif +out:local_irq_restore (save_flags); - objp =cache_alloc_debugcheck_after (cachep, flags, objp, caller); -prefetchw (objp); + ptr =cache_alloc_debugcheck_after (cachep, flags, ptr, caller); +prefetchw (ptr); init =slab_want_init_on_alloc (flags, cachep); -out: -slab_post_alloc_hook (cachep, objcg, flags,1 , &objp, init); -return objp; +out_hooks: +slab_post_alloc_hook (cachep, objcg, flags,1 , &ptr, init); +return ptr; +} + +static __always_inline void * +slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +{ +return slab_alloc_node (cachep, flags,NUMA_NO_NODE , orig_size, caller); }/*
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 01/15] mm/slab: cleanup slab_alloc() and slab_alloc_node()
Quoted 17 lines · Hyeonggon Yoo
mm/slab.c +5
-2+ static __always_inline void * -slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +slab_alloc_node (struct kmem_cache *cachep,gfp_t flags,int nodeid,size_t orig_size, +unsigned long caller) {unsigned long save_flags; -void *objp; +void *ptr; +int slab_node =numa_mem_id ();struct obj_cgroup *objcg =NULL ;bool init = false;@@ -3299,21 +3255,49 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned loif (unlikely (!cachep))return NULL ;
mm/slab.c slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo · line 3308
Quoted 10 lines
mm/slab.c +3
-4- objp =kfence_alloc (cachep, orig_size, flags); -if (unlikely (objp)) - goto out; + ptr =kfence_alloc (cachep, orig_size, flags); +if (unlikely (ptr)) + goto out_hooks;cache_alloc_debugcheck_before (cachep, flags);local_irq_save (save_flags); - objp =__do_cache_alloc (cachep, flags);
Looks like after this patch, slab_alloc() (without a node specified) will not end up in __do_cache_alloc() anymore, so there's no more possibility of alternate_node_alloc(), which looks like a functional regression?
Quoted 45 lines
mm/slab.c +34
-5+ +if (node_match (nodeid, slab_node)) { +/* +* Use the locally cached objects if possible. +* However ____cache_alloc does not allow fallback +* to other nodes. It may fail while we still have +* objects on other nodes available. +*/ + ptr =____cache_alloc (cachep, flags); +if (ptr) + goto out; + } +#ifdef CONFIG_NUMA + else if (unlikely(!get_node(cachep, nodeid))) { +/* Node not bootstrapped yet */ + ptr =fallback_alloc (cachep, flags); + goto out; + } + +/* ___cache_alloc_node can fall back to other nodes */ + ptr =____cache_alloc_node (cachep, flags, nodeid); +#endif +out:local_irq_restore (save_flags); - objp =cache_alloc_debugcheck_after (cachep, flags, objp, caller); -prefetchw (objp); + ptr =cache_alloc_debugcheck_after (cachep, flags, ptr, caller); +prefetchw (ptr); init =slab_want_init_on_alloc (flags, cachep); -out: -slab_post_alloc_hook (cachep, objcg, flags,1 , &objp, init); -return objp; +out_hooks: +slab_post_alloc_hook (cachep, objcg, flags,1 , &ptr, init); +return ptr; +} + +static __always_inline void * +slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +{ +return slab_alloc_node (cachep, flags,NUMA_NO_NODE , orig_size, caller); }/*
Re: [RFC PATCH v1 01/15] mm/slab: cleanup slab_alloc() and slab_alloc_node()
Quoted 25 lines · Vlastimil
Quoted 24 lines · Hyeonggon Yoo
mm/slab.c +8
-5+ static __always_inline void * -slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +slab_alloc_node (struct kmem_cache *cachep,gfp_t flags,int nodeid,size_t orig_size, +unsigned long caller) {unsigned long save_flags; -void *objp; +void *ptr; +int slab_node =numa_mem_id ();struct obj_cgroup *objcg =NULL ;bool init = false;@@ -3299,21 +3255,49 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned loif (unlikely (!cachep))return NULL ; - objp =kfence_alloc (cachep, orig_size, flags); -if (unlikely (objp)) - goto out; + ptr =kfence_alloc (cachep, orig_size, flags); +if (unlikely (ptr)) + goto out_hooks;
Quoted 9 lines
mm/slab.c slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo · line 3308
Quoted 3 lines
mm/slab.c +0
-1cache_alloc_debugcheck_before (cachep, flags);local_irq_save (save_flags); - objp =__do_cache_alloc (cachep, flags);Looks like after this patch,
slab_alloc()(without a node specified) will not end up in__do_cache_alloc()anymore, so there's no more possibility ofalternate_node_alloc(), which looks like a functional regression?
Ah, that was not intended. Thank you for catching this! Will fix in v2.
Thank you so much.
Quoted 44 lines
Quoted 44 lines
mm/slab.c +34
-5+ +if (node_match (nodeid, slab_node)) { +/* +* Use the locally cached objects if possible. +* However ____cache_alloc does not allow fallback +* to other nodes. It may fail while we still have +* objects on other nodes available. +*/ + ptr =____cache_alloc (cachep, flags); +if (ptr) + goto out; + } +#ifdef CONFIG_NUMA + else if (unlikely(!get_node(cachep, nodeid))) { +/* Node not bootstrapped yet */ + ptr =fallback_alloc (cachep, flags); + goto out; + } + +/* ___cache_alloc_node can fall back to other nodes */ + ptr =____cache_alloc_node (cachep, flags, nodeid); +#endif +out:local_irq_restore (save_flags); - objp =cache_alloc_debugcheck_after (cachep, flags, objp, caller); -prefetchw (objp); + ptr =cache_alloc_debugcheck_after (cachep, flags, ptr, caller); +prefetchw (ptr); init =slab_want_init_on_alloc (flags, cachep); -out: -slab_post_alloc_hook (cachep, objcg, flags,1 , &objp, init); -return objp; +out_hooks: +slab_post_alloc_hook (cachep, objcg, flags,1 , &ptr, init); +return ptr; +} + +static __always_inline void * +slab_alloc (struct kmem_cache *cachep,gfp_t flags,size_t orig_size,unsigned long caller) +{ +return slab_alloc_node (cachep, flags,NUMA_NO_NODE , orig_size, caller); }
Quoted 1 line
Review slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo · line 3303
Quoted 1 line
(quoted diff) +0
-0/*
--
Thank you, You are awesome!
Hyeonggon :-)
RFC PATCH 2/15 mm/sl[auo]b: remove CONFIG_NUMA ifdefs for common functions
In reply to Hyeonggon YooNow that slab_alloc_node() is available regardless of CONFIG_NUMA on SLAB, just remove CONFIG_NUMA ifdefs and make non-NUMA version of functions wrapper of NUMA version.
This makes slab allocators use NUMA version of tracepoints. In later patch, tracepoints will be also cleaned up.
Remove now unused __do_kmalloc() in SLAB.
--- include/linux/slab.h | 85 ++++++++++++++++++-------------------------- mm/slab.c | 63 -------------------------------- mm/slob.c | 22 ------------ mm/slub.c | 62 -------------------------------- 4 files changed, 35 insertions(+), 197 deletions(-)
include/linux/slab.h +35 -50
@@ -414,8 +414,31 @@ static __always_inline unsigned int __kmalloc_index(size_t size,#define kmalloc_index (s) __kmalloc_index(s, true)#endif /* !CONFIG_SLOB */ -void *__kmalloc (size_t size,gfp_t flags) __assume_kmalloc_alignment__alloc_size(1) ; -void *kmem_cache_alloc (struct kmem_cache *s,gfp_t flags) __assume_slab_alignment __malloc; +void *__kmalloc_node (size_t size,gfp_t flags,int node) __assume_kmalloc_alignment +__alloc_size(1) ; +void *kmem_cache_alloc_node (struct kmem_cache *s,gfp_t flags,int node) __assume_slab_alignment + __malloc; + +static __always_inline void *__kmalloc(size_t size, gfp_t flags) +{ +return __kmalloc_node (size, flags,NUMA_NO_NODE ); +} + +/** +* kmem_cache_alloc - Allocate an object +* @cachep: The cache to allocate from. +* @flags: See kmalloc(). +* +* Allocate an object from this cache. The flags are only relevant +* if the cache has no available objects. +* +* Return: pointer to the new object or %NULL in case of error +*/ +static __always_inline void *kmem_cache_alloc(struct kmem_cache *s, gfp_t flags) +{ +return kmem_cache_alloc_node (s, flags,NUMA_NO_NODE ); +} +void kmem_cache_free (struct kmem_cache *s,void *objp);/*
@@ -437,38 +460,13 @@ static __always_inline void kfree_bulk(size_t size, void **p)kmem_cache_free_bulk (NULL , size, p); } -#ifdef CONFIG_NUMA -void *__kmalloc_node (size_t size,gfp_t flags,int node) __assume_kmalloc_alignment -__alloc_size(1) ; -void *kmem_cache_alloc_node (struct kmem_cache *s,gfp_t flags,int node) __assume_slab_alignment - __malloc; -#else -static __always_inline __alloc_size(1) void *__kmalloc_node(size_t size, gfp_t flags, int node) -{ -return __kmalloc (size, flags); -} - -static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) -{ -return kmem_cache_alloc (s, flags); -} -#endif -#ifdef CONFIG_TRACING extern void *kmem_cache_alloc_trace (struct kmem_cache *s,gfp_t flags,size_t size) __assume_slab_alignment__alloc_size(3) ; -#ifdef CONFIG_NUMA extern void *kmem_cache_alloc_node_trace (struct kmem_cache *s,gfp_t gfpflags,int node,size_t size) __assume_slab_alignment__alloc_size(4) ; -#else -static __always_inline __alloc_size(4) void *kmem_cache_alloc_node_trace(struct kmem_cache *s, -gfp_t gfpflags,int node,size_t size) -{ -return kmem_cache_alloc_trace (s, gfpflags, size); -} -#endif /* CONFIG_NUMA */ #else /* CONFIG_TRACING */ static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s,
@@ -652,19 +650,6 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flagreturn kmalloc_array (n, size, flags | __GFP_ZERO); } -/* -* kmalloc_track_caller is a special version of kmalloc that records the -* calling function of the routine calling it for slab leak tracking instead -* of just the calling function (confusing, eh?). -* It's useful when the call to kmalloc comes from a widely-used standard -* allocator where we care about the real place the memory allocation -* request comes from. -*/ -extern void *__kmalloc_track_caller (size_t size,gfp_t flags,unsigned long caller) -__alloc_size(1) ; -#define kmalloc_track_caller (size, flags) \ - __kmalloc_track_caller(size, flags, _RET_IP_) -static inline __alloc_size(1, 2) void *kmalloc_array_node (size_t n,size_t size,gfp_t flags,int node) {
@@ -682,21 +667,21 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_treturn kmalloc_array_node (n, size, flags | __GFP_ZERO, node); } - -#ifdef CONFIG_NUMA extern void *__kmalloc_node_track_caller (size_t size,gfp_t flags,int node,unsigned long caller)__alloc_size(1) ;#define kmalloc_node_track_caller (size, flags, node) \ __kmalloc_node_track_caller(size, flags, node, \ _RET_IP_) - -#else /* CONFIG_NUMA */ - -#define kmalloc_node_track_caller (size, flags, node) \ - kmalloc_track_caller(size, flags) - -#endif /* CONFIG_NUMA */ - +/* +* kmalloc_track_caller is a special version of kmalloc that records the +* calling function of the routine calling it for slab leak tracking instead +* of just the calling function (confusing, eh?). +* It's useful when the call to kmalloc comes from a widely-used standard +* allocator where we care about the real place the memory allocation +* request comes from. +*/ +#define kmalloc_track_caller (size, flags) \ + __kmalloc_node_track_caller(size, flags, NUMA_NO_NODE, _RET_IP_)/* * Shortcuts */
mm/slab.c +0 -63
87 lines not on this page · raw message
mm/slob.c +0 -22
@@ -522,26 +522,12 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)return ret; } -void *__kmalloc (size_t size,gfp_t gfp) -{ -return __do_kmalloc_node (size, gfp,NUMA_NO_NODE , _RET_IP_); -} -EXPORT_SYMBOL (__kmalloc); - -void *__kmalloc_track_caller (size_t size,gfp_t gfp,unsigned long caller) -{ -return __do_kmalloc_node (size, gfp,NUMA_NO_NODE , caller); -} -EXPORT_SYMBOL (__kmalloc_track_caller); - -#ifdef CONFIG_NUMA void *__kmalloc_node_track_caller (size_t size,gfp_t gfp,int node,unsigned long caller) {return __do_kmalloc_node (size, gfp, node, caller); }EXPORT_SYMBOL (__kmalloc_node_track_caller); -#endif void kfree (const void *block) {
@@ -629,13 +615,6 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)return b; } -void *kmem_cache_alloc (struct kmem_cache *cachep,gfp_t flags) -{ -return slob_alloc_node (cachep, flags,NUMA_NO_NODE ); -} -EXPORT_SYMBOL (kmem_cache_alloc); - -#ifdef CONFIG_NUMA void *__kmalloc_node (size_t size,gfp_t gfp,int node) {return __do_kmalloc_node (size, gfp, node, _RET_IP_);
@@ -647,7 +626,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)return slob_alloc_node (cachep, gfp, node); }EXPORT_SYMBOL (kmem_cache_alloc_node); -#endif static void __kmem_cache_free (void *b,int size) {
mm/slub.c +0 -62
104 lines not on this page · raw message
Signature · 1 line
2.33.1
RFC PATCH 3/15 mm/sl[au]b: remove CONFIG_TRACING ifdefs for tracing functions
In reply to Hyeonggon YooCONFIG_TRACING ifdefs are not necessary because tracepoints do nothing on kernels without CONFIG_TRACING.
In later cleanup these functions will be removed.
--- include/linux/slab.h | 29 ----------------------------- mm/slab.c | 4 ---- mm/slab_common.c | 2 -- mm/slub.c | 4 ---- 4 files changed, 39 deletions(-)
include/linux/slab.h +0 -29
@@ -460,7 +460,6 @@ static __always_inline void kfree_bulk(size_t size, void **p)kmem_cache_free_bulk (NULL , size, p); } -#ifdef CONFIG_TRACING extern void *kmem_cache_alloc_trace (struct kmem_cache *s,gfp_t flags,size_t size) __assume_slab_alignment__alloc_size(3) ;
@@ -468,39 +467,11 @@ extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) __assume_slab_alignment__alloc_size(4) ; -#else /* CONFIG_TRACING */ -static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s, -gfp_t flags,size_t size) -{ -void *ret =kmem_cache_alloc (s, flags); - - ret =kasan_kmalloc (s, ret, size, flags); -return ret; -} - -static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, -int node,size_t size) -{ -void *ret =kmem_cache_alloc_node (s, gfpflags, node); - - ret =kasan_kmalloc (s, ret, size, gfpflags); -return ret; -} -#endif /* CONFIG_TRACING */ -extern void *kmalloc_order (size_t size,gfp_t flags,unsigned int order) __assume_page_alignment__alloc_size(1) ; -#ifdef CONFIG_TRACING extern void *kmalloc_order_trace (size_t size,gfp_t flags,unsigned int order) __assume_page_alignment__alloc_size(1) ; -#else -static __always_inline __alloc_size(1) void *kmalloc_order_trace(size_t size, gfp_t flags, -unsigned int order) -{ -return kmalloc_order (size, flags, order); -} -#endif static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags) {
mm/slab.c +0 -4
@@ -3519,7 +3519,6 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, }EXPORT_SYMBOL (kmem_cache_alloc_bulk); -#ifdef CONFIG_TRACING void *kmem_cache_alloc_trace (struct kmem_cache *cachep,gfp_t flags,size_t size) {
@@ -3533,7 +3532,6 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)return ret; }EXPORT_SYMBOL (kmem_cache_alloc_trace); -#endif /** * kmem_cache_alloc_node - Allocate an object on the specified node
@@ -3560,7 +3558,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) }EXPORT_SYMBOL (kmem_cache_alloc_node); -#ifdef CONFIG_TRACING void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, gfp_t flags, int nodeid,
@@ -3577,7 +3574,6 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,return ret; }EXPORT_SYMBOL (kmem_cache_alloc_node_trace); -#endif static __always_inline void *__do_kmalloc_node (size_t size,gfp_t flags,int node,unsigned long caller)
mm/slab_common.c +0 -2
@@ -954,7 +954,6 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) }EXPORT_SYMBOL (kmalloc_order); -#ifdef CONFIG_TRACING void *kmalloc_order_trace (size_t size,gfp_t flags,unsigned int order) {void *ret =kmalloc_order (size, flags, order);
@@ -962,7 +961,6 @@ void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)return ret; }EXPORT_SYMBOL (kmalloc_order_trace); -#endif #ifdef CONFIG_SLAB_FREELIST_RANDOM /* Randomize a generic freelist */
mm/slub.c +0 -4
@@ -3238,7 +3238,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,return slab_alloc_node (s, gfpflags,NUMA_NO_NODE , addr, orig_size); } -#ifdef CONFIG_TRACING void *kmem_cache_alloc_trace (struct kmem_cache *s,gfp_t gfpflags,size_t size) {void *ret =slab_alloc (s, gfpflags, _RET_IP_, size);
@@ -3247,7 +3246,6 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)return ret; }EXPORT_SYMBOL (kmem_cache_alloc_trace); -#endif void *kmem_cache_alloc_node (struct kmem_cache *s,gfp_t gfpflags,int node) {
@@ -3260,7 +3258,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) }EXPORT_SYMBOL (kmem_cache_alloc_node); -#ifdef CONFIG_TRACING void *kmem_cache_alloc_node_trace (struct kmem_cache *s,gfp_t gfpflags,int node,size_t size)
@@ -3274,7 +3271,6 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s,return ret; }EXPORT_SYMBOL (kmem_cache_alloc_node_trace); -#endif /* * Slow path handling. This may still be called frequently since objects
Signature · 1 line
2.33.1
RFC PATCH 4/15 mm/sl[auo]b: fold kmalloc_order() into kmalloc_large()
In reply to Hyeonggon YooThere is no caller of kmalloc_order() except kmalloc_large(). Fold it into kmalloc_large() and remove kmalloc_order{,_trace}().
--- include/linux/slab.h | 13 ++----------- mm/slab_common.c | 12 +++--------- 2 files changed, 5 insertions(+), 20 deletions(-)
include/linux/slab.h +2 -11
@@ -467,17 +467,8 @@ extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) __assume_slab_alignment__alloc_size(4) ; -extern void *kmalloc_order (size_t size,gfp_t flags,unsigned int order) __assume_page_alignment -__alloc_size(1) ; - -extern void *kmalloc_order_trace (size_t size,gfp_t flags,unsigned int order) - __assume_page_alignment__alloc_size(1) ; - -static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags) -{ -unsigned int order =get_order (size); -return kmalloc_order_trace (size, flags, order); -} +extern void *kmalloc_large (size_t size,gfp_t flags) __assume_page_alignment +__alloc_size(1) ;/** * kmalloc - allocate memory
mm/slab_common.c +3 -9
@@ -932,10 +932,11 @@ gfp_t kmalloc_fix_flags(gfp_t flags)* directly to the page allocator. We use __GFP_COMP, because we will need to * know the allocation order to free the pages properly in kfree. */ -void *kmalloc_order (size_t size,gfp_t flags,unsigned int order) +void *kmalloc_large (size_t size,gfp_t flags) {void *ret =NULL ;struct page *page; +unsigned int order =get_order (size);if (unlikely (flags &GFP_SLAB_BUG_MASK )) flags =kmalloc_fix_flags (flags);
@@ -950,17 +951,10 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) ret =kasan_kmalloc_large (ret, size, flags);/* As ret might get tagged, call kmemleak hook after KASAN. */ kmemleak_alloc (ret, size,1 , flags); -return ret; -} -EXPORT_SYMBOL (kmalloc_order); - -void *kmalloc_order_trace (size_t size,gfp_t flags,unsigned int order) -{ -void *ret =kmalloc_order (size, flags, order);trace_kmalloc (_RET_IP_, ret, size,PAGE_SIZE << order, flags);return ret; } -EXPORT_SYMBOL (kmalloc_order_trace); +EXPORT_SYMBOL (kmalloc_large);#ifdef CONFIG_SLAB_FREELIST_RANDOM /* Randomize a generic freelist */
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 04/15] mm/sl[auo]b: fold kmalloc_order() into kmalloc_large()
subject prefix could be mm/slab_common
Quoted 4 lines · Hyeonggon Yoo
There is no caller of
kmalloc_order()exceptkmalloc_large(). Fold it intokmalloc_large()and remove kmalloc_order{,_trace}().Signed-off-by Hyeonggon Yoo <[email protected]>
RFC PATCH 5/15 mm/slub: move kmalloc_large_node() to slab_common.c
In reply to Hyeonggon YooIn later patch SLAB will also pass requests larger than order-1 page to page allocator. Move kmalloc_large_node() to slab_common.c.
Fold kmalloc_large_node_hook() into kmalloc_large_node() as there is no other caller.
Move tracepoint in kmalloc_large_node().
Add flag fix code. This exist in kmalloc_large() but omitted in kmalloc_large_node().
--- include/linux/slab.h | 3 +++ mm/slab_common.c | 26 ++++++++++++++++++++++++ mm/slub.c | 47 ++++---------------------------------------- 3 files changed, 33 insertions(+), 43 deletions(-)
include/linux/slab.h +3 -0
@@ -470,6 +470,9 @@ extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,extern void *kmalloc_large (size_t size,gfp_t flags) __assume_page_alignment__alloc_size(1) ; +extern void *kmalloc_large_node (size_t size,gfp_t flags,int node) + __assume_page_alignment__alloc_size(1) ; +/** * kmalloc - allocate memory * @size: how many bytes of memory are required.
mm/slab_common.c +26 -0
@@ -956,6 +956,32 @@ void *kmalloc_large(size_t size, gfp_t flags) }EXPORT_SYMBOL (kmalloc_large); +void *kmalloc_large_node (size_t size,gfp_t flags,int node) +{ +struct page *page; +void *ptr =NULL ; +unsigned int order =get_order (size); + +if (unlikely (flags &GFP_SLAB_BUG_MASK )) + flags =kmalloc_fix_flags (flags); + + flags |= __GFP_COMP; + page =alloc_pages_node (node, flags, order); +if (page) { + ptr =page_address (page); +mod_lruvec_page_state (page,NR_SLAB_UNRECLAIMABLE_B , +PAGE_SIZE << order); + } + ptr =kasan_kmalloc_large (ptr, size, flags); +/* As ptr might get tagged, call kmemleak hook after KASAN. */ +kmemleak_alloc (ptr, size,1 , flags); +trace_kmalloc_node (_RET_IP_, ptr, size,PAGE_SIZE << order, flags, + node); +return ptr; + +} +EXPORT_SYMBOL (kmalloc_large_node); +#ifdef CONFIG_SLAB_FREELIST_RANDOM /* Randomize a generic freelist */ static void freelist_randomize(struct rnd_state *state, unsigned int *list,
mm/slub.c +4 -43
70 lines not on this page · raw message
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 05/15] mm/slub: move kmalloc_large_node() to slab_common.c
Quoted 18 lines · Hyeonggon Yoo
In later patch SLAB will also pass requests larger than order-1 page to page allocator. Move
kmalloc_large_node()toslab_common.c.Fold
kmalloc_large_node_hook()intokmalloc_large_node()as there is no other caller.Move tracepoint in
kmalloc_large_node().Add flag fix code. This exist in
kmalloc_large()but omitted inkmalloc_large_node().Signed-off-by Hyeonggon Yoo <[email protected]>--- include/linux/slab.h | 3 +++ mm/slab_common.c | 26 ++++++++++++++++++++++++ mm/slub.c | 47 ++++---------------------------------------- 3 files changed, 33 insertions(+), 43 deletions(-)
<snip>
mm/slub.c void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, · line 4882
Quoted 11 lines
mm/slub.c +0
-6@@ -4874,15 +4842,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,struct kmem_cache *s;void *ret; -if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) { - ret =kmalloc_large_node (size, gfpflags, node); - -trace_kmalloc_node (caller, ret, - size,PAGE_SIZE <<get_order (size), - gfpflags, node);
Hmm this throws away the caller for tracing, so looks like an unintended functional change.
Quoted 8 lines
mm/slub.c +2
-3- -return ret; - } +if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) +return kmalloc_large_node (size, gfpflags, node); s =kmalloc_slab (size, gfpflags);
RFC PATCH 6/15 mm/slab_common: cleanup kmalloc_large()
In reply to Hyeonggon YooNow that kmalloc_large() and kmalloc_large_node() do same job, make kmalloc_large() wrapper of kmalloc_large_node().
This makes slab allocators to use kmalloc_node tracepoint in kmalloc_large().
--- include/linux/slab.h | 8 +++++--- mm/slab_common.c | 24 ------------------------ 2 files changed, 5 insertions(+), 27 deletions(-)
include/linux/slab.h +5 -3
@@ -467,12 +467,14 @@ extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) __assume_slab_alignment__alloc_size(4) ; -extern void *kmalloc_large (size_t size,gfp_t flags) __assume_page_alignment -__alloc_size(1) ; -extern void *kmalloc_large_node (size_t size,gfp_t flags,int node) __assume_page_alignment__alloc_size(1) ; +static __always_inline void *kmalloc_large(size_t size, gfp_t flags) +{ +return kmalloc_large_node (size, flags,NUMA_NO_NODE ); +} +/** * kmalloc - allocate memory * @size: how many bytes of memory are required.
mm/slab_common.c +0 -24
@@ -932,30 +932,6 @@ gfp_t kmalloc_fix_flags(gfp_t flags)* directly to the page allocator. We use __GFP_COMP, because we will need to * know the allocation order to free the pages properly in kfree. */ -void *kmalloc_large (size_t size,gfp_t flags) -{ -void *ret =NULL ; -struct page *page; -unsigned int order =get_order (size); - -if (unlikely (flags &GFP_SLAB_BUG_MASK )) - flags =kmalloc_fix_flags (flags); - - flags |= __GFP_COMP; - page =alloc_pages (flags, order); -if (likely (page)) { - ret =page_address (page); -mod_lruvec_page_state (page,NR_SLAB_UNRECLAIMABLE_B , -PAGE_SIZE << order); - } - ret =kasan_kmalloc_large (ret, size, flags); -/* As ret might get tagged, call kmemleak hook after KASAN. */ -kmemleak_alloc (ret, size,1 , flags); -trace_kmalloc (_RET_IP_, ret, size,PAGE_SIZE << order, flags); -return ret; -} -EXPORT_SYMBOL (kmalloc_large); -void *kmalloc_large_node (size_t size,gfp_t flags,int node) {struct page *page;
Signature · 1 line
2.33.1
RFC PATCH 7/15 mm/sl[au]b: kmalloc_node: pass large requests to page allocator
In reply to Hyeonggon YooNow that kmalloc_large_node() is in common code, pass large requests to page allocator in kmalloc_node() using kmalloc_large_node().
--- include/linux/slab.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)
include/linux/slab.h +18 -6
@@ -551,23 +551,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)return __kmalloc (size, flags); } +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) { -#ifndef CONFIG_SLOB -if (__builtin_constant_p (size) && - size <=KMALLOC_MAX_CACHE_SIZE ) { -unsigned int i =kmalloc_index (size); +if (__builtin_constant_p (size)) { +unsigned int index; -if (!i) +if (size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large_node (size, flags, node); + + index =kmalloc_index (size); + +if (!index)return ZERO_SIZE_PTR ;return kmem_cache_alloc_node_trace ( kmalloc_caches[kmalloc_type (flags)][i], flags, node, size); } -#endif return __kmalloc_node (size, flags, node); } +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large_node (size, flags, node); + +return __kmalloc_node (size, flags, node); +} +#endif /** * kmalloc_array - allocate memory for an array.
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 07/15] mm/sl[au]b: kmalloc_node: pass large requests to page allocator
Quoted 4 lines · Hyeonggon Yoo
Now that
kmalloc_large_node()is in common code, pass large requests to page allocator inkmalloc_node()usingkmalloc_large_node().Signed-off-by Hyeonggon Yoo <[email protected]>
Quoted 50 lines
--- include/linux/slab.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)include/linux/slab.h +18
-6@@ -551,23 +551,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)return __kmalloc (size, flags); } +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) { -#ifndef CONFIG_SLOB -if (__builtin_constant_p (size) && - size <=KMALLOC_MAX_CACHE_SIZE ) { -unsigned int i =kmalloc_index (size); +if (__builtin_constant_p (size)) { +unsigned int index; -if (!i) +if (size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large_node (size, flags, node); + + index =kmalloc_index (size); + +if (!index)return ZERO_SIZE_PTR ;return kmem_cache_alloc_node_trace ( kmalloc_caches[kmalloc_type (flags)][i], flags, node, size); } -#endif return __kmalloc_node (size, flags, node); } +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large_node (size, flags, node); + +return __kmalloc_node (size, flags, node); +} +#endif /** * kmalloc_array - allocate memory for an array.
RFC PATCH 8/15 mm/sl[auo]b: cleanup kmalloc()
In reply to Hyeonggon YooNow that kmalloc() and kmalloc_node() do same job, make kmalloc() wrapper of kmalloc_node().
Remove kmalloc_trace() that is now unused. This patch makes slab allocator use kmalloc_node tracepoints in kmalloc().
--- include/linux/slab.h | 82 +++++++++++++++++--------------------------- mm/slab.c | 14 -------- mm/slub.c | 9 ----- 3 files changed, 31 insertions(+), 74 deletions(-)
include/linux/slab.h +31 -51
@@ -460,9 +460,6 @@ static __always_inline void kfree_bulk(size_t size, void **p)kmem_cache_free_bulk (NULL , size, p); } -extern void *kmem_cache_alloc_trace (struct kmem_cache *s,gfp_t flags,size_t size) - __assume_slab_alignment__alloc_size(3) ; -extern void *kmem_cache_alloc_node_trace (struct kmem_cache *s,gfp_t gfpflags,int node,size_t size) __assume_slab_alignment__alloc_size(4) ;
@@ -475,6 +472,36 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)return kmalloc_large_node (size, flags,NUMA_NO_NODE ); } +#ifndef CONFIG_SLOB +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size)) { +unsigned int index; + +if (size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags); + + index =kmalloc_index (size); + +if (!index) +return ZERO_SIZE_PTR ; + +return kmem_cache_alloc_node_trace ( + kmalloc_caches[kmalloc_type (flags)][index], + flags, node, size); + } +return __kmalloc_node (size, flags, node); +} +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags); + +return __kmalloc_node (size, flags, node); +} +#endif +/** * kmalloc - allocate memory * @size: how many bytes of memory are required.
@@ -531,55 +558,8 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)*/ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) { -if (__builtin_constant_p (size)) { -#ifndef CONFIG_SLOB -unsigned int index; -#endif -if (size >KMALLOC_MAX_CACHE_SIZE ) -return kmalloc_large (size, flags); -#ifndef CONFIG_SLOB - index =kmalloc_index (size); - -if (!index) -return ZERO_SIZE_PTR ; - -return kmem_cache_alloc_trace ( - kmalloc_caches[kmalloc_type (flags)][index], - flags, size); -#endif - } -return __kmalloc (size, flags); -} - -#ifndef CONFIG_SLOB -static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) -{ -if (__builtin_constant_p (size)) { -unsigned int index; - -if (size >KMALLOC_MAX_CACHE_SIZE ) -return kmalloc_large_node (size, flags, node); - - index =kmalloc_index (size); - -if (!index) -return ZERO_SIZE_PTR ; - -return kmem_cache_alloc_node_trace ( - kmalloc_caches[kmalloc_type (flags)][i], - flags, node, size); - } -return __kmalloc_node (size, flags, node); -} -#else -static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) -{ -if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) -return kmalloc_large_node (size, flags, node); - -return __kmalloc_node (size, flags, node); +return kmalloc_node (size, flags,NUMA_NO_NODE ); } -#endif /** * kmalloc_array - allocate memory for an array.
mm/slab.c +0 -14
@@ -3519,20 +3519,6 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, }EXPORT_SYMBOL (kmem_cache_alloc_bulk); -void * -kmem_cache_alloc_trace (struct kmem_cache *cachep,gfp_t flags,size_t size) -{ -void *ret; - - ret =slab_alloc (cachep, flags, size, _RET_IP_); - - ret =kasan_kmalloc (cachep, ret, size, flags); -trace_kmalloc (_RET_IP_, ret, - size, cachep->size, flags); -return ret; -} -EXPORT_SYMBOL (kmem_cache_alloc_trace); -/** * kmem_cache_alloc_node - Allocate an object on the specified node * @cachep: The cache to allocate from.
mm/slub.c +0 -9
@@ -3230,15 +3230,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,return slab_alloc_node (s, gfpflags,NUMA_NO_NODE , addr, orig_size); } -void *kmem_cache_alloc_trace (struct kmem_cache *s,gfp_t gfpflags,size_t size) -{ -void *ret =slab_alloc (s, gfpflags, _RET_IP_, size); -trace_kmalloc (_RET_IP_, ret, size, s->size, gfpflags); - ret =kasan_kmalloc (s, ret, size, gfpflags); -return ret; -} -EXPORT_SYMBOL (kmem_cache_alloc_trace); -void *kmem_cache_alloc_node (struct kmem_cache *s,gfp_t gfpflags,int node) {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, s->object_size);
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 08/15] mm/sl[auo]b: cleanup kmalloc()
Quoted 4 lines · Hyeonggon Yoo
Now that
kmalloc()andkmalloc_node()do same job, makekmalloc()wrapper ofkmalloc_node().Remove
kmalloc_trace()that is now unused. This patch makes slab
You mean remove kmem_cache_alloc_trace().
Quoted 3 lines
allocator use
kmalloc_nodetracepoints inkmalloc().Signed-off-by Hyeonggon Yoo <[email protected]>
Re: [RFC PATCH v1 08/15] mm/sl[auo]b: cleanup kmalloc()
In reply to Hyeonggon YooQuoted 7 lines · Hyeonggon Yoo
Now that
kmalloc()andkmalloc_node()do same job, makekmalloc()wrapper ofkmalloc_node().Remove
kmalloc_trace()that is now unused. This patch makes slab allocator usekmalloc_nodetracepoints inkmalloc().Signed-off-by Hyeonggon Yoo <[email protected]>
Actually there are more things to fix than the commit log.
include/linux/slab.h static __always_inline void *kmalloc_large(size_t size, gfp_t flags) · line 482
Quoted 8 lines
include/linux/slab.h +8
-0+#ifndef CONFIG_SLOB +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size)) { +unsigned int index; + +if (size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags);
Should use kmalloc_large_node().
Quoted 7 lines
include/linux/slab.h +7
-0+ + index =kmalloc_index (size); + +if (!index) +return ZERO_SIZE_PTR ; + + return kmem_cache_alloc_node_trace(
include/linux/slab.h static __always_inline void *kmalloc_large(size_t size, gfp_t flags) · line 499
Quoted 10 lines
include/linux/slab.h +10
-0+ kmalloc_caches[kmalloc_type (flags)][index], + flags, node, size); + } +return __kmalloc_node (size, flags, node); +} +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags);
And here.
Quoted 4 lines
include/linux/slab.h +4
-0+ + return __kmalloc_node(size, flags, node); +} +#endif
Re: [RFC PATCH v1 08/15] mm/sl[auo]b: cleanup kmalloc()
Quoted 33 lines · Vlastimil Babka
Quoted 7 lines · Hyeonggon Yoo
Now that
kmalloc()andkmalloc_node()do same job, makekmalloc()wrapper ofkmalloc_node().Remove
kmalloc_trace()that is now unused. This patch makes slab allocator usekmalloc_nodetracepoints inkmalloc().Signed-off-by Hyeonggon Yoo <[email protected]>Actually there are more things to fix than the commit log.
include/linux/slab.h static __always_inline void *kmalloc_large(size_t size, gfp_t flags) · line 482
Quoted 8 lines
include/linux/slab.h +8
-0+#ifndef CONFIG_SLOB +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size)) { +unsigned int index; + +if (size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags);Should use
kmalloc_large_node().Quoted 11 lines
include/linux/slab.h +11
-0+ + index =kmalloc_index (size); + +if (!index) +return ZERO_SIZE_PTR ; + +return kmem_cache_alloc_node_trace ( + kmalloc_caches[kmalloc_type (flags)][index], + flags, node, size); + } +return __kmalloc_node (size, flags, node);
Quoted 9 lines
include/linux/slab.h static __always_inline void *kmalloc_large(size_t size, gfp_t flags) · line 499
Quoted 6 lines
include/linux/slab.h +6
-0+} +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags);And here.
Sorry for the mistake. did it in v2. thanks!
Quoted 4 lines
Quoted 4 lines
include/linux/slab.h +4
-0+ + return __kmalloc_node(size, flags, node); +} +#endif
Signature · 2 lines
Thanks, Hyeonggon
RFC PATCH 9/15 mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
In reply to Hyeonggon YooThere is not much benefit for serving large objects in kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.
[ [email protected]: Enable and disable irq around free_large_kmalloc().
Do not lose NUMA locality in __do_kmalloc_node().
Use folio_slab(folio)->slab_cache instead of virt_to_cache(). Remove large sizes in __kmalloc_index(). ]
--- include/linux/slab.h | 23 +++++----------------- mm/slab.c | 45 ++++++++++++++++++++++++++++++-------------- mm/slab.h | 3 +++ mm/slab_common.c | 25 +++++++++++++++++------- mm/slub.c | 19 ------------------- 5 files changed, 57 insertions(+), 58 deletions(-)
include/linux/slab.h +5 -18
@@ -226,27 +226,17 @@ void kmem_dump_obj(void *object);#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to -* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW
@@ -398,10 +388,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size,if (size <=512 *1024 )return 19 ;if (size <=1024 *1024 )return 20 ;if (size <=2 *1024 *1024 )return 21 ; -if (size <=4 *1024 *1024 )return 22 ; -if (size <=8 *1024 *1024 )return 23 ; -if (size <=16 *1024 *1024 )return 24 ; -if (size <=32 *1024 *1024 )return 25 ;if (!IS_ENABLED (CONFIG_PROFILE_ALL_BRANCHES ) && size_is_constant)BUILD_BUG_ON_MSG (1 ,"unexpected size in kmalloc_index()" );
@@ -411,6 +397,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size,/* Will never be reached. Needed because the compiler may complain */ return -1; } +static_assert (PAGE_SHIFT <=20 );#define kmalloc_index (s) __kmalloc_index(s, true)#endif /* !CONFIG_SLOB */
mm/slab.c +31 -14
88 lines not on this page · raw message
mm/slab.h +3 -0
@@ -664,6 +664,9 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)print_tracking (cachep, x); return cachep; } + +void free_large_kmalloc (struct folio *folio,void *object); +#endif /* CONFIG_SLOB */ static inline size_t slab_ksize (const struct kmem_cache *s)
mm/slab_common.c +18 -7
@@ -759,8 +759,8 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)/* * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. -* kmalloc_index() supports up to 2^25=32MB, so the final entry of the table is -* kmalloc-32M. +* kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is +* kmalloc-2M. */ const struct kmalloc_info_struct kmalloc_info[]__initconst = {INIT_KMALLOC_INFO (0 ,0 ),
@@ -784,11 +784,7 @@ const struct kmalloc_info_struct kmalloc_info[] __initconst = { INIT_KMALLOC_INFO(262144, 256k), INIT_KMALLOC_INFO(524288, 512k), INIT_KMALLOC_INFO(1048576, 1M), - INIT_KMALLOC_INFO(2097152, 2M), - INIT_KMALLOC_INFO(4194304, 4M), - INIT_KMALLOC_INFO(8388608, 8M), - INIT_KMALLOC_INFO(16777216, 16M), - INIT_KMALLOC_INFO(33554432, 32M) + INIT_KMALLOC_INFO(2097152, 2M) };/*
@@ -913,6 +909,21 @@ void __init create_kmalloc_caches(slab_flags_t flags) }#endif } + +void free_large_kmalloc (struct folio *folio,void *object) +{ +unsigned int order =folio_order (folio); + +if (WARN_ON_ONCE (order ==0 )) +pr_warn_once ("object pointer: 0x%p\n" , object); + +kmemleak_free (object); +kasan_kfree_large (object); + +mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , + -(PAGE_SIZE << order)); +__free_pages (folio_page (folio,0 ), order); +}#endif /* !CONFIG_SLOB */ gfp_t kmalloc_fix_flags (gfp_t flags)
mm/slub.c +0 -19
@@ -1678,12 +1678,6 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,* Hooks for other subsystems that check memory allocations. In a typical * production configuration these hooks all should produce no code at all. */ -static __always_inline void kfree_hook(void *x) -{ -kmemleak_free (x); -kasan_kfree_large (x); -} - static __always_inline bool slab_free_hook(struct kmem_cache *s,void *x,bool init) {
@@ -3501,19 +3495,6 @@ struct detached_freelist {struct kmem_cache *s; }; -static inline void free_large_kmalloc (struct folio *folio,void *object) -{ -unsigned int order =folio_order (folio); - -if (WARN_ON_ONCE (order ==0 )) -pr_warn_once ("object pointer: 0x%p\n" , object); - -kfree_hook (object); -mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , - -(PAGE_SIZE << order)); -__free_pages (folio_page (folio,0 ), order); -} -/* * This function progressively scans the array with free objects (with * a limited look ahead) and extract objects belonging to the same
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 09/15] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
Quoted 8 lines · Hyeonggon Yoo
There is not much benefit for serving large objects in
kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.[ [email protected]: Enable and disable irq around
free_large_kmalloc().
Do not lose NUMA locality in__do_kmalloc_node().
Use folio_slab(folio)->slab_cache instead ofvirt_to_cache(). Remove large sizes in__kmalloc_index(). ]
Thanks for the mention but that's generally only done like this if I took your patch and made those changes myself. But I just suggested them. Small suggested changes like this are usually just mentioned in e.g. v1->v2 changelogs.
Quoted 69 lines
Signed-off-by Hyeonggon Yoo <[email protected]>--- include/linux/slab.h | 23 +++++----------------- mm/slab.c | 45 ++++++++++++++++++++++++++++++-------------- mm/slab.h | 3 +++ mm/slab_common.c | 25 +++++++++++++++++------- mm/slub.c | 19 ------------------- 5 files changed, 57 insertions(+), 58 deletions(-)include/linux/slab.h +5
-18@@ -226,27 +226,17 @@ void kmem_dump_obj(void *object);#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to -* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW @@ -398,10 +388,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size,if (size <=512 *1024 )return 19 ;if (size <=1024 *1024 )return 20 ;if (size <=2 *1024 *1024 )return 21 ; -if (size <=4 *1024 *1024 )return 22 ; -if (size <=8 *1024 *1024 )return 23 ; -if (size <=16 *1024 *1024 )return 24 ; -if (size <=32 *1024 *1024 )return 25 ;if (!IS_ENABLED (CONFIG_PROFILE_ALL_BRANCHES ) && size_is_constant)BUILD_BUG_ON_MSG (1 ,"unexpected size in kmalloc_index()" );@@ -411,6 +397,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size,/* Will never be reached. Needed because the compiler may complain */ return -1; } +static_assert (PAGE_SHIFT <=20 );#define kmalloc_index (s) __kmalloc_index(s, true)#endif /* !CONFIG_SLOB */ mm/slab.c +0
-0
mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3571
Quoted 5 lines
mm/slab.c +1
-1@@ -3568,7 +3568,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large_node (size, flags, node);
Similar issue with caller not traced.
Quoted 33 lines
mm/slab.c +13
-3cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3642,15 +3642,25 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) {struct kmem_cache *s;size_t i; +struct folio *folio;local_irq_disable ();for (i =0 ; i < size; i++) {void *objp = p[i]; -if (!orig_s)/* called via kfree_bulk */ - s =virt_to_cache (objp); -else +if (!orig_s) { + folio =virt_to_folio (objp); +/* called via kfree_bulk */ +if (!folio_test_slab (folio)) { +local_irq_enable (); +free_large_kmalloc (folio, objp); +local_irq_disable (); +continue ; + } + s =folio_slab (folio)->slab_cache; + }else s =cache_from_obj (orig_s, objp); +if (!s)continue ;
mm/slab.c void kfree(const void *objp) · line 3693
Quoted 5 lines
mm/slab.c +2
-0@@ -3679,20 +3689,25 @@ void kfree(const void *objp) {struct kmem_cache *c;unsigned long flags; +struct folio *folio; +void *x = (void *) objp;
I think you don't need to add 'x', just do the cast while calling free_large_kmalloc(), same as done for __cache_free().
Quoted 26 lines
mm/slab.c +9
-6trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ; -local_irq_save (flags); -kfree_debugcheck (objp); - c =virt_to_cache (objp); -if (!c) { -local_irq_restore (flags); + + folio =virt_to_folio (objp); +if (!folio_test_slab (folio)) { +free_large_kmalloc (folio, x);return ; } -debug_check_no_locks_freed (objp, c->object_size); + c =folio_slab (folio)->slab_cache; + +local_irq_save (flags); +kfree_debugcheck (objp); +debug_check_no_locks_freed (objp, c->object_size);debug_check_no_obj_freed (objp, c->object_size);__cache_free (c, (void *)objp, _RET_IP_);local_irq_restore (flags);
Re: [RFC PATCH v1 09/15] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
Quoted 9 lines · Vlastimil Babka
Quoted 8 lines · Hyeonggon Yoo
There is not much benefit for serving large objects in
kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.[ [email protected]: Enable and disable irq around
free_large_kmalloc().
Do not lose NUMA locality in__do_kmalloc_node().
Use folio_slab(folio)->slab_cache instead ofvirt_to_cache(). Remove large sizes in__kmalloc_index(). ]
A bit late to reply but better late than never...
Quoted 5 lines
Thanks for the mention but that's generally only done like this if I took your patch and made those changes myself. But I just suggested them. Small suggested changes like this are usually just mentioned in e.g.
v1->v2changelogs.
Ah, okay. I didn't know about the convention. thanks for letting me know!
Quoted 71 lines
Quoted 71 lines
Signed-off-by Hyeonggon Yoo <[email protected]>--- include/linux/slab.h | 23 +++++----------------- mm/slab.c | 45 ++++++++++++++++++++++++++++++-------------- mm/slab.h | 3 +++ mm/slab_common.c | 25 +++++++++++++++++------- mm/slub.c | 19 ------------------- 5 files changed, 57 insertions(+), 58 deletions(-)include/linux/slab.h +5
-18@@ -226,27 +226,17 @@ void kmem_dump_obj(void *object);#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to -* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW @@ -398,10 +388,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size,if (size <=512 *1024 )return 19 ;if (size <=1024 *1024 )return 20 ;if (size <=2 *1024 *1024 )return 21 ; -if (size <=4 *1024 *1024 )return 22 ; -if (size <=8 *1024 *1024 )return 23 ; -if (size <=16 *1024 *1024 )return 24 ; -if (size <=32 *1024 *1024 )return 25 ;if (!IS_ENABLED (CONFIG_PROFILE_ALL_BRANCHES ) && size_is_constant)BUILD_BUG_ON_MSG (1 ,"unexpected size in kmalloc_index()" );@@ -411,6 +397,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size,/* Will never be reached. Needed because the compiler may complain */ return -1; } +static_assert (PAGE_SHIFT <=20 );#define kmalloc_index (s) __kmalloc_index(s, true)#endif /* !CONFIG_SLOB */ mm/slab.c +0
-0@@ -3568,7 +3568,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *ret;
Quoted 6 lines
mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3571
Quoted 3 lines
mm/slab.c +1
-1if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large_node (size, flags, node);Similar issue with caller not traced.
Actually I moved tracepoint into kmalloc_large_node(), but the problem I think was I write patches hard to review.
in v2 I split some patches to be more reviewable. thanks!!
Quoted 32 lines
Quoted 32 lines
mm/slab.c +13
-3cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3642,15 +3642,25 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) {struct kmem_cache *s;size_t i; +struct folio *folio;local_irq_disable ();for (i =0 ; i < size; i++) {void *objp = p[i]; -if (!orig_s)/* called via kfree_bulk */ - s =virt_to_cache (objp); -else +if (!orig_s) { + folio =virt_to_folio (objp); +/* called via kfree_bulk */ +if (!folio_test_slab (folio)) { +local_irq_enable (); +free_large_kmalloc (folio, objp); +local_irq_disable (); +continue ; + } + s =folio_slab (folio)->slab_cache; + }else s =cache_from_obj (orig_s, objp); +if (!s)continue ;
Quoted 10 lines
mm/slab.c void kfree(const void *objp) · line 3693
Quoted 6 lines
mm/slab.c +2
-0@@ -3679,20 +3689,25 @@ void kfree(const void *objp) {struct kmem_cache *c;unsigned long flags; +struct folio *folio; +void *x = (void *) objp;I think you don't need to add 'x', just do the cast while calling
free_large_kmalloc(), same as done for__cache_free().
in fact also SLUB's kfree defines x. But your suggestion sounds better. Anyway did it in v2. thanks!
Quoted 26 lines
Quoted 26 lines
mm/slab.c +9
-6trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ; -local_irq_save (flags); -kfree_debugcheck (objp); - c =virt_to_cache (objp); -if (!c) { -local_irq_restore (flags); + + folio =virt_to_folio (objp); +if (!folio_test_slab (folio)) { +free_large_kmalloc (folio, x);return ; } -debug_check_no_locks_freed (objp, c->object_size); + c =folio_slab (folio)->slab_cache; + +local_irq_save (flags); +kfree_debugcheck (objp); +debug_check_no_locks_freed (objp, c->object_size);debug_check_no_obj_freed (objp, c->object_size);__cache_free (c, (void *)objp, _RET_IP_);local_irq_restore (flags);
Signature · 2 lines
Thanks, Hyeonggon
RFC PATCH 10/15 mm/sl[auo]b: print cache name in tracepoints
In reply to Hyeonggon YooPrint cache name in tracepoints. If there is no corresponding cache (kmalloc in SLOB or kmalloc_large_node), use KMALLOC_{,LARGE_}NAME macro.
--- include/trace/events/kmem.h | 34 +++++++++++++++++++--------------- mm/slab.c | 6 +++--- mm/slab.h | 4 ++++ mm/slab_common.c | 4 ++-- mm/slob.c | 10 +++++----- mm/slub.c | 10 +++++----- 6 files changed, 38 insertions(+), 30 deletions(-)
include/trace/events/kmem.h +19 -15
@@ -61,16 +61,18 @@ DEFINE_EVENT(kmem_alloc, kmem_cache_alloc, DECLARE_EVENT_CLASS(kmem_alloc_node, - TP_PROTO(unsigned long call_site, + TP_PROTO(const char *name, + unsigned long call_site, const void *ptr, size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags, int node), - TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node), + TP_ARGS(name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node), TP_STRUCT__entry( + __string( name, name ) __field( unsigned long, call_site ) __field( const void *, ptr ) __field( size_t, bytes_req )
@@ -80,6 +82,7 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, ), TP_fast_assign( + __assign_str(name, name); __entry->call_site = call_site; __entry->ptr = ptr; __entry->bytes_req = bytes_req;
@@ -88,7 +91,8 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->node = node; ), - TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d", + TP_printk("name=%s call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d", + __get_str(name), (void *)__entry->call_site, __entry->ptr, __entry->bytes_req,
@@ -99,20 +103,20 @@ DECLARE_EVENT_CLASS(kmem_alloc_node,DEFINE_EVENT (kmem_alloc_node, kmalloc_node, - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, + TP_PROTO(const char *name, unsigned long call_site, + const void *ptr, size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags, int node), -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) +TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) );DEFINE_EVENT (kmem_alloc_node, kmem_cache_alloc_node, - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, + TP_PROTO(const char *name, unsigned long call_site, + const void *ptr, size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags, int node), -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) +TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ); TRACE_EVENT(kfree,
@@ -137,24 +141,24 @@ TRACE_EVENT(kfree,TRACE_EVENT (kmem_cache_free, - TP_PROTO(unsigned long call_site, const void *ptr, const char *name), + TP_PROTO(const char *name, unsigned long call_site, const void *ptr), -TP_ARGS (call_site, ptr, name), +TP_ARGS (name, call_site, ptr),TP_STRUCT__entry ( + __string( name, name ) __field( unsigned long, call_site ) __field( const void *, ptr ) -__string ( name, name ) ),TP_fast_assign ( + __assign_str(name, name); __entry->call_site = call_site; __entry->ptr = ptr; - __assign_str(name, name); ), -TP_printk ("call_site=%pS ptr=%p name=%s" , - (void *)__entry->call_site, __entry->ptr,__get_str (name)) +TP_printk ("name=%s call_site=%pS ptr=%p" , +__get_str (name), (void *)__entry->call_site, __entry->ptr) ); TRACE_EVENT(mm_page_free,
mm/slab.c +3 -3
@@ -3536,7 +3536,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) {void *ret =slab_alloc_node (cachep, flags, nodeid, cachep->object_size, _RET_IP_); -trace_kmem_cache_alloc_node (_RET_IP_, ret, +trace_kmem_cache_alloc_node (cachep->name, _RET_IP_, ret, cachep->object_size, cachep->size, flags, nodeid);
@@ -3554,7 +3554,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret =slab_alloc_node (cachep, flags, nodeid, size, _RET_IP_); ret =kasan_kmalloc (cachep, ret, size, flags); -trace_kmalloc_node (_RET_IP_, ret, +trace_kmalloc_node (cachep->name, _RET_IP_, ret, size, cachep->size, flags, nodeid);return ret;
@@ -3628,7 +3628,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)if (!cachep)return ; -trace_kmem_cache_free (_RET_IP_, objp, cachep->name); +trace_kmem_cache_free (cachep->name, _RET_IP_, objp);local_irq_save (flags);debug_check_no_locks_freed (objp, cachep->object_size);if (!(cachep->flags &SLAB_DEBUG_OBJECTS ))
mm/slab.h +4 -0
@@ -274,6 +274,10 @@ void create_kmalloc_caches(slab_flags_t);struct kmem_cache *kmalloc_slab (size_t ,gfp_t );#endif +/* cache names for tracepoints where it has no corresponding cache */ +#define KMALLOC_LARGE_NAME "kmalloc_large_node" +#define KMALLOC_NAME "kmalloc_node" +gfp_t kmalloc_fix_flags (gfp_t flags);/* Functions provided by the slab allocators */
mm/slab_common.c +2 -2
@@ -962,8 +962,8 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) ptr =kasan_kmalloc_large (ptr, size, flags);/* As ptr might get tagged, call kmemleak hook after KASAN. */ kmemleak_alloc (ptr, size,1 , flags); -trace_kmalloc_node (_RET_IP_, ptr, size,PAGE_SIZE << order, flags, - node); +trace_kmalloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, +PAGE_SIZE << order, flags, node);return ptr; }
mm/slob.c +5 -5
@@ -505,7 +505,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) *m = size; ret = (void *)m + minalign; -trace_kmalloc_node (caller, ret, +trace_kmalloc_node (KMALLOC_NAME , caller, ret, size, size + minalign, gfp, node); }else {unsigned int order =get_order (size);
@@ -514,7 +514,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) gfp |= __GFP_COMP; ret =slob_new_pages (gfp, order, node); -trace_kmalloc_node (caller, ret, +trace_kmalloc_node (KMALLOC_LARGE_NAME , caller, ret, size,PAGE_SIZE << order, gfp, node); }
@@ -596,12 +596,12 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)if (c->size <PAGE_SIZE ) { b =slob_alloc (c->size, flags, c->align, node,0 ); -trace_kmem_cache_alloc_node (_RET_IP_, b, c->object_size, +trace_kmem_cache_alloc_node (c->name, _RET_IP_, b, c->object_size,SLOB_UNITS (c->size) *SLOB_UNIT , flags, node); }else { b =slob_new_pages (flags,get_order (c->size), node); -trace_kmem_cache_alloc_node (_RET_IP_, b, c->object_size, +trace_kmem_cache_alloc_node (c->name, _RET_IP_, b, c->object_size,PAGE_SIZE <<get_order (c->size), flags, node); }
@@ -646,7 +646,7 @@ static void kmem_rcu_free(struct rcu_head *head)void kmem_cache_free (struct kmem_cache *c,void *b) {kmemleak_free_recursive (b, c->flags); -trace_kmem_cache_free (_RET_IP_, b, c->name); +trace_kmem_cache_free (c->name, _RET_IP_, b);if (unlikely (c->flags &SLAB_TYPESAFE_BY_RCU )) {struct slob_rcu *slob_rcu; slob_rcu = b + (c->size -sizeof (struct slob_rcu ));
mm/slub.c +5 -5
@@ -3228,7 +3228,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, s->object_size); -trace_kmem_cache_alloc_node (_RET_IP_, ret, +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, s->object_size, s->size, gfpflags, node);return ret;
@@ -3241,7 +3241,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, size); -trace_kmalloc_node (_RET_IP_, ret, +trace_kmalloc_node (s->name, _RET_IP_, ret, size, s->size, gfpflags, node); ret =kasan_kmalloc (s, ret, size, gfpflags);
@@ -3482,7 +3482,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) s =cache_from_obj (s, x);if (!s)return ; -trace_kmem_cache_free (_RET_IP_, x, s->name); +trace_kmem_cache_free (s->name, _RET_IP_, x);slab_free (s,virt_to_slab (x), x,NULL ,1 , _RET_IP_); }EXPORT_SYMBOL (kmem_cache_free);
@@ -4366,7 +4366,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ret =slab_alloc_node (s, flags, node, _RET_IP_, size); -trace_kmalloc_node (_RET_IP_, ret, size, s->size, flags, node); +trace_kmalloc_node (s->name, _RET_IP_, ret, size, s->size, flags, node); ret =kasan_kmalloc (s, ret, size, flags);
@@ -4825,7 +4825,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ret =slab_alloc_node (s, gfpflags, node, caller, size);/* Honor the call site pointer we received. */ -trace_kmalloc_node (caller, ret, size, s->size, gfpflags, node); +trace_kmalloc_node (s->name, caller, ret, size, s->size, gfpflags, node);return ret; }
Signature · 1 line
2.33.1
RFC PATCH 11/15 mm/sl[auo]b: use same tracepoint in kmalloc and normal caches
In reply to Hyeonggon YooNow that tracepoints print cache names, we can distinguish kmalloc and normal cache allocations.
Use same tracepoint in kmalloc and normal caches. After this patch, there is only two tracepoints in slab allocators: kmem_cache_alloc_node and kmem_cache_free.
Remove all unused tracepoints.
--- include/trace/events/kmem.h | 79 ------------------------------------- mm/slab.c | 8 ++-- mm/slab_common.c | 5 ++- mm/slob.c | 14 ++++--- mm/slub.c | 19 +++++---- 5 files changed, 27 insertions(+), 98 deletions(-)
include/trace/events/kmem.h +0 -79
@@ -9,56 +9,6 @@#include <linux/tracepoint.h> #include <trace/events/mmflags.h> -DECLARE_EVENT_CLASS (kmem_alloc, - - TP_PROTO(unsigned long call_site, - const void *ptr, - size_t bytes_req, - size_t bytes_alloc, - gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - __field( size_t, bytes_req ) - __field( size_t, bytes_alloc ) -__field ( gfp_t, gfp_flags ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - __entry->bytes_req = bytes_req; - __entry->bytes_alloc = bytes_alloc; - __entry->gfp_flags = gfp_flags; - ), - -TP_printk ("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s" , - (void *)__entry->call_site, - __entry->ptr, - __entry->bytes_req, - __entry->bytes_alloc, -show_gfp_flags (__entry->gfp_flags)) -); - -DEFINE_EVENT (kmem_alloc, kmalloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - -DEFINE_EVENT (kmem_alloc, kmem_cache_alloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - DECLARE_EVENT_CLASS(kmem_alloc_node, TP_PROTO(const char *name,
@@ -101,15 +51,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->node) ); -DEFINE_EVENT (kmem_alloc_node, kmalloc_node, - - TP_PROTO(const char *name, unsigned long call_site, - const void *ptr, size_t bytes_req, size_t bytes_alloc, - gfp_t gfp_flags, int node), - -TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) -); - DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, TP_PROTO(const char *name, unsigned long call_site,
@@ -119,26 +60,6 @@ DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ); -TRACE_EVENT (kfree, - - TP_PROTO(unsigned long call_site, const void *ptr), - -TP_ARGS (call_site, ptr), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - ), - -TP_printk ("call_site=%pS ptr=%p" , - (void *)__entry->call_site, __entry->ptr) -); - TRACE_EVENT(kmem_cache_free, TP_PROTO(const char *name, unsigned long call_site, const void *ptr),
mm/slab.c +4 -4
@@ -3554,9 +3554,9 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret =slab_alloc_node (cachep, flags, nodeid, size, _RET_IP_); ret =kasan_kmalloc (cachep, ret, size, flags); -trace_kmalloc_node (cachep->name, _RET_IP_, ret, - size, cachep->size, - flags, nodeid); +trace_kmem_cache_alloc_node (cachep->name, _RET_IP_, ret, + size, cachep->size, + flags, nodeid);return ret; }EXPORT_SYMBOL (kmem_cache_alloc_node_trace);
@@ -3692,7 +3692,6 @@ void kfree(const void *objp)struct folio *folio;void *x = (void *) objp; -trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ;
@@ -3704,6 +3703,7 @@ void kfree(const void *objp) } c =folio_slab (folio)->slab_cache; +trace_kmem_cache_free (c->name, _RET_IP_, objp);local_irq_save (flags);kfree_debugcheck (objp);
mm/slab_common.c +3 -2
@@ -917,6 +917,7 @@ void free_large_kmalloc(struct folio *folio, void *object)if (WARN_ON_ONCE (order ==0 ))pr_warn_once ("object pointer: 0x%p\n" , object); +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, object);kmemleak_free (object);kasan_kfree_large (object);
@@ -962,8 +963,8 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) ptr =kasan_kmalloc_large (ptr, size, flags);/* As ptr might get tagged, call kmemleak hook after KASAN. */ kmemleak_alloc (ptr, size,1 , flags); -trace_kmalloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, -PAGE_SIZE << order, flags, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, +PAGE_SIZE << order, flags, node);return ptr; }
mm/slob.c +8 -6
@@ -505,8 +505,8 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) *m = size; ret = (void *)m + minalign; -trace_kmalloc_node (KMALLOC_NAME , caller, ret, - size, size + minalign, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_NAME , caller, ret, + size, size + minalign, gfp, node); }else {unsigned int order =get_order (size);
@@ -514,8 +514,9 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) gfp |= __GFP_COMP; ret =slob_new_pages (gfp, order, node); -trace_kmalloc_node (KMALLOC_LARGE_NAME , caller, ret, - size,PAGE_SIZE << order, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , caller, + ret, size,PAGE_SIZE << order, + gfp, node); }kmemleak_alloc (ret, size,1 , gfp);
@@ -533,8 +534,6 @@ void kfree(const void *block) {struct folio *sp; -trace_kfree (_RET_IP_, block); -if (unlikely (ZERO_OR_NULL_PTR (block)))return ;kmemleak_free (block);
@@ -543,10 +542,13 @@ void kfree(const void *block)if (folio_test_slab (sp)) {int align =max_t (size_t,ARCH_KMALLOC_MINALIGN ,ARCH_SLAB_MINALIGN );unsigned int *m = (unsigned int *)(block - align); + +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, block);slob_free (m, *m + align); }else {unsigned int order =folio_order (sp); +trace_kmem_cache_free (KMALLOC_NAME , _RET_IP_, block);mod_node_page_state (folio_pgdat (sp),NR_SLAB_UNRECLAIMABLE_B , -(PAGE_SIZE << order));__free_pages (folio_page (sp,0 ), order);
mm/slub.c +12 -7
@@ -3241,8 +3241,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, - size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, + size, s->size, gfpflags, node); ret =kasan_kmalloc (s, ret, size, gfpflags);return ret;
@@ -4366,7 +4366,8 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ret =slab_alloc_node (s, flags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, size, s->size, flags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, size, + s->size, flags, node); ret =kasan_kmalloc (s, ret, size, flags);
@@ -4445,8 +4446,7 @@ void kfree(const void *x)struct folio *folio;struct slab *slab;void *object = (void *)x; - -trace_kfree (_RET_IP_, x); +struct kmem_cache *s;if (unlikely (ZERO_OR_NULL_PTR (x)))return ;
@@ -4456,8 +4456,12 @@ void kfree(const void *x)free_large_kmalloc (folio, object);return ; } + slab =folio_slab (folio); -slab_free (slab->slab_cache, slab, object,NULL ,1 , _RET_IP_); + s = slab->slab_cache; + +trace_kmem_cache_free (s->name, _RET_IP_, x); +slab_free (s, slab, object,NULL ,1 , _RET_IP_); }EXPORT_SYMBOL (kfree);
@@ -4825,7 +4829,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ret =slab_alloc_node (s, gfpflags, node, caller, size);/* Honor the call site pointer we received. */ -trace_kmalloc_node (s->name, caller, ret, size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, caller, ret, size, + s->size, gfpflags, node);return ret; }
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 11/15] mm/sl[auo]b: use same tracepoint in kmalloc and normal caches
Quoted 6 lines · Hyeonggon Yoo
Now that tracepoints print cache names, we can distinguish kmalloc and normal cache allocations.
Use same tracepoint in kmalloc and normal caches. After this patch, there is only two tracepoints in slab allocators:
kmem_cache_alloc_nodeandkmem_cache_free.
Haven't made up my mind yet about whether the extra overhead of cache names is worth it. But at least the unification of _node and non_node versions make sense. I would however use the tracepoint names without "_node" in them
Quoted 280 lines
Remove all unused tracepoints.
Signed-off-by Hyeonggon Yoo <[email protected]>--- include/trace/events/kmem.h | 79 ------------------------------------- mm/slab.c | 8 ++-- mm/slab_common.c | 5 ++- mm/slob.c | 14 ++++--- mm/slub.c | 19 +++++---- 5 files changed, 27 insertions(+), 98 deletions(-)include/trace/events/kmem.h +0
-79@@ -9,56 +9,6 @@#include <linux/tracepoint.h> #include <trace/events/mmflags.h> -DECLARE_EVENT_CLASS (kmem_alloc, - - TP_PROTO(unsigned long call_site, - const void *ptr, - size_t bytes_req, - size_t bytes_alloc, - gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - __field( size_t, bytes_req ) - __field( size_t, bytes_alloc ) -__field ( gfp_t, gfp_flags ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - __entry->bytes_req = bytes_req; - __entry->bytes_alloc = bytes_alloc; - __entry->gfp_flags = gfp_flags; - ), - -TP_printk ("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s" , - (void *)__entry->call_site, - __entry->ptr, - __entry->bytes_req, - __entry->bytes_alloc, -show_gfp_flags (__entry->gfp_flags)) -); - -DEFINE_EVENT (kmem_alloc, kmalloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - -DEFINE_EVENT (kmem_alloc, kmem_cache_alloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - DECLARE_EVENT_CLASS(kmem_alloc_node, TP_PROTO(const char *name,@@ -101,15 +51,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->node) ); -DEFINE_EVENT (kmem_alloc_node, kmalloc_node, - - TP_PROTO(const char *name, unsigned long call_site, - const void *ptr, size_t bytes_req, size_t bytes_alloc, - gfp_t gfp_flags, int node), - -TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) -); - DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, TP_PROTO(const char *name, unsigned long call_site,@@ -119,26 +60,6 @@ DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ); -TRACE_EVENT (kfree, - - TP_PROTO(unsigned long call_site, const void *ptr), - -TP_ARGS (call_site, ptr), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - ), - -TP_printk ("call_site=%pS ptr=%p" , - (void *)__entry->call_site, __entry->ptr) -); - TRACE_EVENT(kmem_cache_free, TP_PROTO(const char *name, unsigned long call_site, const void *ptr),mm/slab.c +4
-4@@ -3554,9 +3554,9 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret =slab_alloc_node (cachep, flags, nodeid, size, _RET_IP_); ret =kasan_kmalloc (cachep, ret, size, flags); -trace_kmalloc_node (cachep->name, _RET_IP_, ret, - size, cachep->size, - flags, nodeid); +trace_kmem_cache_alloc_node (cachep->name, _RET_IP_, ret, + size, cachep->size, + flags, nodeid);return ret; }EXPORT_SYMBOL (kmem_cache_alloc_node_trace);@@ -3692,7 +3692,6 @@ void kfree(const void *objp)struct folio *folio;void *x = (void *) objp; -trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ;@@ -3704,6 +3703,7 @@ void kfree(const void *objp) } c =folio_slab (folio)->slab_cache; +trace_kmem_cache_free (c->name, _RET_IP_, objp);local_irq_save (flags);kfree_debugcheck (objp);mm/slab_common.c +3
-2@@ -917,6 +917,7 @@ void free_large_kmalloc(struct folio *folio, void *object)if (WARN_ON_ONCE (order ==0 ))pr_warn_once ("object pointer: 0x%p\n" , object); +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, object);kmemleak_free (object);kasan_kfree_large (object);@@ -962,8 +963,8 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) ptr =kasan_kmalloc_large (ptr, size, flags);/* As ptr might get tagged, call kmemleak hook after KASAN. */ kmemleak_alloc (ptr, size,1 , flags); -trace_kmalloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, -PAGE_SIZE << order, flags, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, +PAGE_SIZE << order, flags, node);return ptr; }mm/slob.c +8
-6@@ -505,8 +505,8 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) *m = size; ret = (void *)m + minalign; -trace_kmalloc_node (KMALLOC_NAME , caller, ret, - size, size + minalign, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_NAME , caller, ret, + size, size + minalign, gfp, node); }else {unsigned int order =get_order (size);@@ -514,8 +514,9 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) gfp |= __GFP_COMP; ret =slob_new_pages (gfp, order, node); -trace_kmalloc_node (KMALLOC_LARGE_NAME , caller, ret, - size,PAGE_SIZE << order, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , caller, + ret, size,PAGE_SIZE << order, + gfp, node); }kmemleak_alloc (ret, size,1 , gfp);@@ -533,8 +534,6 @@ void kfree(const void *block) {struct folio *sp; -trace_kfree (_RET_IP_, block); -if (unlikely (ZERO_OR_NULL_PTR (block)))return ;kmemleak_free (block);@@ -543,10 +542,13 @@ void kfree(const void *block)if (folio_test_slab (sp)) {int align =max_t (size_t,ARCH_KMALLOC_MINALIGN ,ARCH_SLAB_MINALIGN );unsigned int *m = (unsigned int *)(block - align); + +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, block);slob_free (m, *m + align); }else {unsigned int order =folio_order (sp); +trace_kmem_cache_free (KMALLOC_NAME , _RET_IP_, block);mod_node_page_state (folio_pgdat (sp),NR_SLAB_UNRECLAIMABLE_B , -(PAGE_SIZE << order));__free_pages (folio_page (sp,0 ), order);mm/slub.c +12
-7@@ -3241,8 +3241,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, - size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, + size, s->size, gfpflags, node); ret =kasan_kmalloc (s, ret, size, gfpflags);return ret;@@ -4366,7 +4366,8 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ret =slab_alloc_node (s, flags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, size, s->size, flags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, size, + s->size, flags, node); ret =kasan_kmalloc (s, ret, size, flags);@@ -4445,8 +4446,7 @@ void kfree(const void *x)struct folio *folio;struct slab *slab;void *object = (void *)x; - -trace_kfree (_RET_IP_, x); +struct kmem_cache *s;if (unlikely (ZERO_OR_NULL_PTR (x)))return ;@@ -4456,8 +4456,12 @@ void kfree(const void *x)free_large_kmalloc (folio, object);return ; } + slab =folio_slab (folio); -slab_free (slab->slab_cache, slab, object,NULL ,1 , _RET_IP_); + s = slab->slab_cache; + +trace_kmem_cache_free (s->name, _RET_IP_, x); +slab_free (s, slab, object,NULL ,1 , _RET_IP_); }EXPORT_SYMBOL (kfree);@@ -4825,7 +4829,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ret =slab_alloc_node (s, gfpflags, node, caller, size);/* Honor the call site pointer we received. */ -trace_kmalloc_node (s->name, caller, ret, size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, caller, ret, size, + s->size, gfpflags, node);return ret; }
Re: [RFC PATCH v1 11/15] mm/sl[auo]b: use same tracepoint in kmalloc and normal caches
Quoted 11 lines · Vlastimil Babka
Quoted 6 lines · Hyeonggon Yoo
Now that tracepoints print cache names, we can distinguish kmalloc and normal cache allocations.
Use same tracepoint in kmalloc and normal caches. After this patch, there is only two tracepoints in slab allocators:
kmem_cache_alloc_nodeandkmem_cache_free.Haven't made up my mind yet about whether the extra overhead of cache names is worth it. But at least the unification of
_nodeandnon_nodeversions make sense.
How much do you expect the overhead of printing cache names? If the overhead is the problem I think it's just okay not to print its name.
even before this series we were not able to distinguish allocation between caches. we could just distinguish them by checking the tracepoint is kmem_cache_alloc or kmalloc.
Quoted 1 line
I would however use the tracepoint names without "
_node" in them
But anyway the suggestion makes sense and I renamed it in v2. thanks!
Quoted 281 lines
Quoted 280 lines
Remove all unused tracepoints.
Signed-off-by Hyeonggon Yoo <[email protected]>--- include/trace/events/kmem.h | 79 ------------------------------------- mm/slab.c | 8 ++-- mm/slab_common.c | 5 ++- mm/slob.c | 14 ++++--- mm/slub.c | 19 +++++---- 5 files changed, 27 insertions(+), 98 deletions(-)include/trace/events/kmem.h +0
-79@@ -9,56 +9,6 @@#include <linux/tracepoint.h> #include <trace/events/mmflags.h> -DECLARE_EVENT_CLASS (kmem_alloc, - - TP_PROTO(unsigned long call_site, - const void *ptr, - size_t bytes_req, - size_t bytes_alloc, - gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - __field( size_t, bytes_req ) - __field( size_t, bytes_alloc ) -__field ( gfp_t, gfp_flags ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - __entry->bytes_req = bytes_req; - __entry->bytes_alloc = bytes_alloc; - __entry->gfp_flags = gfp_flags; - ), - -TP_printk ("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s" , - (void *)__entry->call_site, - __entry->ptr, - __entry->bytes_req, - __entry->bytes_alloc, -show_gfp_flags (__entry->gfp_flags)) -); - -DEFINE_EVENT (kmem_alloc, kmalloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - -DEFINE_EVENT (kmem_alloc, kmem_cache_alloc, - - TP_PROTO(unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - -TP_ARGS (call_site, ptr, bytes_req, bytes_alloc, gfp_flags) -); - DECLARE_EVENT_CLASS(kmem_alloc_node, TP_PROTO(const char *name,@@ -101,15 +51,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->node) ); -DEFINE_EVENT (kmem_alloc_node, kmalloc_node, - - TP_PROTO(const char *name, unsigned long call_site, - const void *ptr, size_t bytes_req, size_t bytes_alloc, - gfp_t gfp_flags, int node), - -TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) -); - DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, TP_PROTO(const char *name, unsigned long call_site,@@ -119,26 +60,6 @@ DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,TP_ARGS (name, call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ); -TRACE_EVENT (kfree, - - TP_PROTO(unsigned long call_site, const void *ptr), - -TP_ARGS (call_site, ptr), - -TP_STRUCT__entry ( - __field( unsigned long, call_site ) - __field( const void *, ptr ) - ), - -TP_fast_assign ( - __entry->call_site = call_site; - __entry->ptr = ptr; - ), - -TP_printk ("call_site=%pS ptr=%p" , - (void *)__entry->call_site, __entry->ptr) -); - TRACE_EVENT(kmem_cache_free, TP_PROTO(const char *name, unsigned long call_site, const void *ptr),mm/slab.c +4
-4@@ -3554,9 +3554,9 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, ret =slab_alloc_node (cachep, flags, nodeid, size, _RET_IP_); ret =kasan_kmalloc (cachep, ret, size, flags); -trace_kmalloc_node (cachep->name, _RET_IP_, ret, - size, cachep->size, - flags, nodeid); +trace_kmem_cache_alloc_node (cachep->name, _RET_IP_, ret, + size, cachep->size, + flags, nodeid);return ret; }EXPORT_SYMBOL (kmem_cache_alloc_node_trace);@@ -3692,7 +3692,6 @@ void kfree(const void *objp)struct folio *folio;void *x = (void *) objp; -trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ;@@ -3704,6 +3703,7 @@ void kfree(const void *objp) } c =folio_slab (folio)->slab_cache; +trace_kmem_cache_free (c->name, _RET_IP_, objp);local_irq_save (flags);kfree_debugcheck (objp);mm/slab_common.c +3
-2@@ -917,6 +917,7 @@ void free_large_kmalloc(struct folio *folio, void *object)if (WARN_ON_ONCE (order ==0 ))pr_warn_once ("object pointer: 0x%p\n" , object); +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, object);kmemleak_free (object);kasan_kfree_large (object);@@ -962,8 +963,8 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) ptr =kasan_kmalloc_large (ptr, size, flags);/* As ptr might get tagged, call kmemleak hook after KASAN. */ kmemleak_alloc (ptr, size,1 , flags); -trace_kmalloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, -PAGE_SIZE << order, flags, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , _RET_IP_, ptr, size, +PAGE_SIZE << order, flags, node);return ptr; }mm/slob.c +8
-6@@ -505,8 +505,8 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) *m = size; ret = (void *)m + minalign; -trace_kmalloc_node (KMALLOC_NAME , caller, ret, - size, size + minalign, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_NAME , caller, ret, + size, size + minalign, gfp, node); }else {unsigned int order =get_order (size);@@ -514,8 +514,9 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) gfp |= __GFP_COMP; ret =slob_new_pages (gfp, order, node); -trace_kmalloc_node (KMALLOC_LARGE_NAME , caller, ret, - size,PAGE_SIZE << order, gfp, node); +trace_kmem_cache_alloc_node (KMALLOC_LARGE_NAME , caller, + ret, size,PAGE_SIZE << order, + gfp, node); }kmemleak_alloc (ret, size,1 , gfp);@@ -533,8 +534,6 @@ void kfree(const void *block) {struct folio *sp; -trace_kfree (_RET_IP_, block); -if (unlikely (ZERO_OR_NULL_PTR (block)))return ;kmemleak_free (block);@@ -543,10 +542,13 @@ void kfree(const void *block)if (folio_test_slab (sp)) {int align =max_t (size_t,ARCH_KMALLOC_MINALIGN ,ARCH_SLAB_MINALIGN );unsigned int *m = (unsigned int *)(block - align); + +trace_kmem_cache_free (KMALLOC_LARGE_NAME , _RET_IP_, block);slob_free (m, *m + align); }else {unsigned int order =folio_order (sp); +trace_kmem_cache_free (KMALLOC_NAME , _RET_IP_, block);mod_node_page_state (folio_pgdat (sp),NR_SLAB_UNRECLAIMABLE_B , -(PAGE_SIZE << order));__free_pages (folio_page (sp,0 ), order);mm/slub.c +12
-7@@ -3241,8 +3241,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, {void *ret =slab_alloc_node (s, gfpflags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, - size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, + size, s->size, gfpflags, node); ret =kasan_kmalloc (s, ret, size, gfpflags);return ret;@@ -4366,7 +4366,8 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ret =slab_alloc_node (s, flags, node, _RET_IP_, size); -trace_kmalloc_node (s->name, _RET_IP_, ret, size, s->size, flags, node); +trace_kmem_cache_alloc_node (s->name, _RET_IP_, ret, size, + s->size, flags, node); ret =kasan_kmalloc (s, ret, size, flags);@@ -4445,8 +4446,7 @@ void kfree(const void *x)struct folio *folio;struct slab *slab;void *object = (void *)x; - -trace_kfree (_RET_IP_, x); +struct kmem_cache *s;if (unlikely (ZERO_OR_NULL_PTR (x)))return ;@@ -4456,8 +4456,12 @@ void kfree(const void *x)free_large_kmalloc (folio, object);return ; } + slab =folio_slab (folio); -slab_free (slab->slab_cache, slab, object,NULL ,1 , _RET_IP_); + s = slab->slab_cache; + +trace_kmem_cache_free (s->name, _RET_IP_, x); +slab_free (s, slab, object,NULL ,1 , _RET_IP_); }EXPORT_SYMBOL (kfree);@@ -4825,7 +4829,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ret =slab_alloc_node (s, gfpflags, node, caller, size);/* Honor the call site pointer we received. */ -trace_kmalloc_node (s->name, caller, ret, size, s->size, gfpflags, node); +trace_kmem_cache_alloc_node (s->name, caller, ret, size, + s->size, gfpflags, node);return ret; }
Signature · 2 lines
Thanks, Hyeonggon
RFC PATCH 12/15 mm/sl[au]b: generalize kmalloc subsystem
In reply to Hyeonggon YooNow everything in kmalloc subsystem can be generalized. Let's do it!
Generalize __kmalloc_node_track_caller(), kfree(), __ksize(), and move them to slab_common.c.
Make __kmalloc_node() wrapper of __kmalloc_node_track_caller(). They are duplicate.
To keep caller address unchanged in kmalloc/kfree tracepoints, implement __kmem_cache_{alloc_node,free}() that takes caller address.
--- include/linux/slab.h | 79 ++++++++++++++++++------- mm/slab.c | 135 ++++--------------------------------------- mm/slab_common.c | 75 ++++++++++++++++++++++++ mm/slob.c | 32 +++++----- mm/slub.c | 105 +++------------------------------ 5 files changed, 166 insertions(+), 260 deletions(-)
include/linux/slab.h +57 -22
@@ -401,14 +401,53 @@ static_assert(PAGE_SHIFT <= 20);#define kmalloc_index (s) __kmalloc_index(s, true)#endif /* !CONFIG_SLOB */ -void *__kmalloc_node (size_t size,gfp_t flags,int node) __assume_kmalloc_alignment -__alloc_size(1) ; -void *kmem_cache_alloc_node (struct kmem_cache *s,gfp_t flags,int node) __assume_slab_alignment - __malloc; +extern void *__kmalloc_node_track_caller (size_t size,gfp_t flags,int node, +unsigned long caller)__alloc_size(1) ; +#define kmalloc_node_track_caller (size, flags, node) \ + __kmalloc_node_track_caller(size, flags, node, \ + _RET_IP_) +/* +* kmalloc_track_caller is a special version of kmalloc that records the +* calling function of the routine calling it for slab leak tracking instead +* of just the calling function (confusing, eh?). +* It's useful when the call to kmalloc comes from a widely-used standard +* allocator where we care about the real place the memory allocation +* request comes from. +*/ +#define kmalloc_track_caller (size, flags) \ + __kmalloc_node_track_caller(size, flags, NUMA_NO_NODE, _RET_IP_) + +static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ +return __kmalloc_node_track_caller (size, flags, node, _RET_IP_); +} static __always_inline void *__kmalloc(size_t size, gfp_t flags) { -return __kmalloc_node (size, flags,NUMA_NO_NODE ); +return __kmalloc_node_track_caller (size, flags,NUMA_NO_NODE , _RET_IP_); +} + +void __kmem_cache_free (struct kmem_cache *s,void *x,unsigned long caller); +void *__kmem_cache_alloc_node (struct kmem_cache *s,gfp_t gfpflags, +int node,unsigned long caller); + +/** +* kmem_cache_alloc_node - Allocate an object on the specified node +* @cachep: The cache to allocate from. +* @flags: See kmalloc(). +* @nodeid: node number of the target node. +* +* Identical to kmem_cache_alloc but it will allocate memory on the given +* node, which can improve the performance for cpu bound structures. +* +* Fallback to other node is possible if __GFP_THISNODE is not set. +* +* Return: pointer to the new object or %NULL in case of error +*/ +static __always_inline void * +kmem_cache_alloc_node (struct kmem_cache *s,gfp_t gfpflags,int node) +{ +return __kmem_cache_alloc_node (s, gfpflags, node, _RET_IP_); }/**
@@ -423,10 +462,21 @@ static __always_inline void *__kmalloc(size_t size, gfp_t flags)*/ static __always_inline void *kmem_cache_alloc(struct kmem_cache *s, gfp_t flags) { -return kmem_cache_alloc_node (s, flags,NUMA_NO_NODE ); +return __kmem_cache_alloc_node (s, flags,NUMA_NO_NODE , _RET_IP_); } -void kmem_cache_free (struct kmem_cache *s,void *objp); +/** +* kmem_cache_free - Deallocate an object +* @cachep: The cache the allocation was from. +* @objp: The previously allocated object. +* +* Free an object which was previously allocated from this +* cache. +*/ +static __always_inline void kmem_cache_free(struct kmem_cache *s, void *x) +{ +__kmem_cache_free (s, x, _RET_IP_); +}/* * Bulk allocation and freeing operations. These are accelerated in an
@@ -613,21 +663,6 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_treturn kmalloc_array_node (n, size, flags | __GFP_ZERO, node); } -extern void *__kmalloc_node_track_caller (size_t size,gfp_t flags,int node, -unsigned long caller)__alloc_size(1) ; -#define kmalloc_node_track_caller (size, flags, node) \ - __kmalloc_node_track_caller(size, flags, node, \ - _RET_IP_) -/* -* kmalloc_track_caller is a special version of kmalloc that records the -* calling function of the routine calling it for slab leak tracking instead -* of just the calling function (confusing, eh?). -* It's useful when the call to kmalloc comes from a widely-used standard -* allocator where we care about the real place the memory allocation -* request comes from. -*/ -#define kmalloc_track_caller (size, flags) \ - __kmalloc_node_track_caller(size, flags, NUMA_NO_NODE, _RET_IP_)/* * Shortcuts */
mm/slab.c +11 -124
181 lines not on this page · raw message
mm/slab_common.c +75 -0
81 lines not on this page · raw message
mm/slob.c +14 -18
@@ -588,7 +588,8 @@ int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)return 0 ; } -static void *slob_alloc_node (struct kmem_cache *c,gfp_t flags,int node) +static void *slob_alloc_node (struct kmem_cache *c,gfp_t flags, +int node,unsigned long caller) {void *b;
@@ -598,12 +599,12 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)if (c->size <PAGE_SIZE ) { b =slob_alloc (c->size, flags, c->align, node,0 ); -trace_kmem_cache_alloc_node (c->name, _RET_IP_, b, c->object_size, +trace_kmem_cache_alloc_node (c->name, caller, b, c->object_size,SLOB_UNITS (c->size) *SLOB_UNIT , flags, node); }else { b =slob_new_pages (flags,get_order (c->size), node); -trace_kmem_cache_alloc_node (c->name, _RET_IP_, b, c->object_size, +trace_kmem_cache_alloc_node (c->name, caller, b, c->object_size,PAGE_SIZE <<get_order (c->size), flags, node); }
@@ -617,19 +618,14 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)return b; } -void *__kmalloc_node (size_t size,gfp_t gfp,int node) +void *__kmem_cache_alloc_node (struct kmem_cache *cachep,gfp_t gfp, +int node,unsigned long caller) { -return __do_kmalloc_node (size, gfp, node, _RET_IP_); +return slob_alloc_node (cachep, gfp, node, caller); } -EXPORT_SYMBOL (__kmalloc_node); +EXPORT_SYMBOL (__kmem_cache_alloc_node); -void *kmem_cache_alloc_node (struct kmem_cache *cachep,gfp_t gfp,int node) -{ -return slob_alloc_node (cachep, gfp, node); -} -EXPORT_SYMBOL (kmem_cache_alloc_node); - -static void __kmem_cache_free (void *b,int size) +static void ___kmem_cache_free (void *b,int size) {if (size <PAGE_SIZE )slob_free (b, size);
@@ -642,23 +638,23 @@ static void kmem_rcu_free(struct rcu_head *head)struct slob_rcu *slob_rcu = (struct slob_rcu *)head;void *b = (void *)slob_rcu - (slob_rcu->size -sizeof (struct slob_rcu )); -__kmem_cache_free (b, slob_rcu->size); +___kmem_cache_free (b, slob_rcu->size); } -void kmem_cache_free (struct kmem_cache *c,void *b) +void __kmem_cache_free (struct kmem_cache *c,void *b,unsigned long caller) {kmemleak_free_recursive (b, c->flags); -trace_kmem_cache_free (c->name, _RET_IP_, b); +trace_kmem_cache_free (c->name, caller, b);if (unlikely (c->flags &SLAB_TYPESAFE_BY_RCU )) {struct slob_rcu *slob_rcu; slob_rcu = b + (c->size -sizeof (struct slob_rcu )); slob_rcu->size = c->size;call_rcu (&slob_rcu->head, kmem_rcu_free); }else { -__kmem_cache_free (b, c->size); +___kmem_cache_free (b, c->size); } } -EXPORT_SYMBOL (kmem_cache_free); +EXPORT_SYMBOL (__kmem_cache_free);void kmem_cache_free_bulk (struct kmem_cache *s,size_t size,void **p) {
mm/slub.c +9 -96
146 lines not on this page · raw message
Signature · 1 line
2.33.1
RFC PATCH 13/15 mm/sl[au]b: remove kmem_cache_alloc_node_trace()
In reply to Hyeonggon Yookmem_cache_alloc_node_trace() was introduced by commit 4a92379bdfb4 ("slub tracing: move trace calls out of always inlined functions to reduce kernel code size") to avoid inlining tracepoints for inlined kmalloc function calls.
Now that we use same tracepoint in kmalloc and normal caches, kmem_cache_alloc_node_trace() can be replaced with __kmem_cache_alloc_node() and kasan_kmalloc().
--- include/linux/slab.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-)
include/linux/slab.h +8 -7
@@ -497,10 +497,6 @@ static __always_inline void kfree_bulk(size_t size, void **p)kmem_cache_free_bulk (NULL , size, p); } -extern void *kmem_cache_alloc_node_trace (struct kmem_cache *s,gfp_t gfpflags, -int node,size_t size) __assume_slab_alignment -__alloc_size(4) ; -extern void *kmalloc_large_node (size_t size,gfp_t flags,int node) __assume_page_alignment__alloc_size(1) ;
@@ -512,6 +508,9 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) { +struct kmem_cache *s; +void *objp; +if (__builtin_constant_p (size)) {unsigned int index;
@@ -523,9 +522,11 @@ static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flaif (!index)return ZERO_SIZE_PTR ; -return kmem_cache_alloc_node_trace ( - kmalloc_caches[kmalloc_type (flags)][index], - flags, node, size); + s = kmalloc_caches[kmalloc_type (flags)][index]; + + objp =__kmem_cache_alloc_node (s, flags, node, _RET_IP_); + objp =kasan_kmalloc (s, objp, size, flags); +return objp; }return __kmalloc_node (size, flags, node); }
Signature · 1 line
2.33.1
RFC PATCH 14/15 mm/sl[auo]b: move definition of __ksize() to mm/slab.h
In reply to Hyeonggon Yoo__ksize() is only called by KASAN. Remove export symbol and move definition to mm/slab.h as we don't want to grow its callers.
[ [email protected]: Move definition to mm/slab.h and reduce comments ]
--- include/linux/slab.h | 1 - mm/slab.h | 2 ++ mm/slab_common.c | 11 +---------- mm/slob.c | 1 - 4 files changed, 3 insertions(+), 12 deletions(-)
include/linux/slab.h +0 -1
@@ -182,7 +182,6 @@ int kmem_cache_shrink(struct kmem_cache *s);void *__must_check krealloc (const void *objp,size_t new_size,gfp_t flags)__alloc_size(2) ;void kfree (const void *objp);void kfree_sensitive (const void *objp); -size_t __ksize (const void *objp);size_t ksize (const void *objp);#ifdef CONFIG_PRINTK bool kmem_valid_obj (void *object);
mm/slab.h +2 -0
@@ -673,6 +673,8 @@ void free_large_kmalloc(struct folio *folio, void *object);#endif /* CONFIG_SLOB */ +size_t __ksize (const void *objp); +static inline size_t slab_ksize (const struct kmem_cache *s) {#ifndef CONFIG_SLUB
mm/slab_common.c +1 -10
@@ -976,15 +976,7 @@ void kfree(const void *x) }EXPORT_SYMBOL (kfree); -/** -* __ksize -- Uninstrumented ksize. -* @objp: pointer to the object -* -* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same -* safety checks as ksize() with KASAN instrumentation enabled. -* -* Return: size of the actual memory used by @objp in bytes -*/ +/* Uninstrumented ksize. Only called by KASAN. */ size_t __ksize (const void *object) {struct folio *folio;
@@ -999,7 +991,6 @@ size_t __ksize(const void *object) return slab_ksize(folio_slab(folio)->slab_cache); } -EXPORT_SYMBOL (__ksize);#endif /* !CONFIG_SLOB */ gfp_t kmalloc_fix_flags (gfp_t flags)
mm/slob.c +0 -1
@@ -576,7 +576,6 @@ size_t __ksize(const void *block) m = (unsigned int *)(block - align);return SLOB_UNITS (*m) *SLOB_UNIT ; } -EXPORT_SYMBOL (__ksize);int __kmem_cache_create (struct kmem_cache *c,slab_flags_t flags) {
Signature · 1 line
2.33.1
RFC PATCH 15/15 mm/sl[au]b: check if large object is valid in __ksize()
In reply to Hyeonggon Yoo__ksize() returns size of objects allocated from slab allocator. When invalid object is passed to __ksize(), returning zero prevents further memory corruption and makes caller be able to check if there is an error.
If address of large object is not beginning of folio or size of the folio is too small, it must be invalid. Return zero in such cases.
--- mm/slab_common.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
mm/slab_common.c +5 -1
@@ -986,8 +986,12 @@ size_t __ksize(const void *object) folio =virt_to_folio (object); -if (unlikely (!folio_test_slab (folio))) +if (unlikely (!folio_test_slab (folio))) { +if (object !=folio_address (folio) || +folio_size (folio) <=KMALLOC_MAX_CACHE_SIZE ) +return 0 ;return folio_size (folio); + }return slab_ksize (folio_slab (folio)->slab_cache); }
Signature · 1 line
2.33.1
Re: [RFC PATCH v1 15/15] mm/sl[au]b: check if large object is valid in __ksize()
Quoted 25 lines · Vlastimil
Quoted 24 lines · Hyeonggon Yoo
(quoted diff) +8
-5+ static __always_inline void * -slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller) +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size, + unsigned long caller) { unsigned long save_flags; - void *objp; + void *ptr; + int slab_node = numa_mem_id(); struct obj_cgroup *objcg = NULL; bool init = false;@@ -3299,21 +3255,49 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo if (unlikely(!cachep)) return NULL; - objp = kfence_alloc(cachep, orig_size, flags); - if (unlikely(objp)) - goto out; + ptr = kfence_alloc(cachep, orig_size, flags); + if (unlikely(ptr)) + goto out_hooks;
Quoted 9 lines
Review slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo · line 3308
Quoted 3 lines
(quoted diff) +0
-1cache_alloc_debugcheck_before(cachep, flags); local_irq_save(save_flags); - objp = __do_cache_alloc(cachep, flags);Looks like after this patch,
slab_alloc()(without a node specified) will not end up in__do_cache_alloc()anymore, so there's no more possibility ofalternate_node_alloc(), which looks like a functional regression?
Ah, that was not intended. Thank you for catching this! Will fix in v2.
Thank you so much.
Quoted 44 lines
Quoted 44 lines
(quoted diff) +34
-5+ + if (node_match(nodeid, slab_node)) { + /* + * Use the locally cached objects if possible. + * However ____cache_alloc does not allow fallback + * to other nodes. It may fail while we still have + * objects on other nodes available. + */ + ptr = ____cache_alloc(cachep, flags); + if (ptr) + goto out; + } +#ifdef CONFIG_NUMA + else if (unlikely(!get_node(cachep, nodeid))) { + /* Node not bootstrapped yet */ + ptr = fallback_alloc(cachep, flags); + goto out; + } + + /* ___cache_alloc_node can fall back to other nodes */ + ptr = ____cache_alloc_node(cachep, flags, nodeid); +#endif +out: local_irq_restore(save_flags); - objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); - prefetchw(objp); + ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); + prefetchw(ptr); init = slab_want_init_on_alloc(flags, cachep); -out: - slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init); - return objp; +out_hooks: + slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init); + return ptr; +} + +static __always_inline void * +slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller) +{ + return slab_alloc_node(cachep, flags, NUMA_NO_NODE, orig_size, caller); }
Quoted 1 line
Review slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo · line 3303
Quoted 1 line
(quoted diff) +0
-0/*
--
Thank you, You are awesome!
Hyeonggon :-)