slab cleanups
Subsystems (2)
v2 cover slab cleanups
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.
Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.
Hello, these are cleanup patches for slab.
Please consider them for slab-next :)
Any comments will be appreciated.
Thanks.
Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify __ksize()
mm/sl[auo]b: move definition of __ksize() to mm/slab.h mm/slub: limit number of node partial slabs only in cache creation mm/slub: refactor deactivate_slab()
include/linux/slab.h | 36 ++++++------ mm/slab.c | 51 ++++++++--------- mm/slab.h | 21 +++++++ mm/slab_common.c | 20 +++++++ mm/slob.c | 1 - mm/slub.c | 130 ++++++++++++------------------------------- 6 files changed, 114 insertions(+), 145 deletions(-)
Signature · 1 line
2.33.1
PATCH v2 1/5 mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
There is not much benefit for serving large objects in kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.
--- include/linux/slab.h | 35 ++++++++++++++++------------------- mm/slab.c | 31 +++++++++++++++++++++++++++---- mm/slab.h | 19 +++++++++++++++++++ mm/slub.c | 19 ------------------- 4 files changed, 62 insertions(+), 42 deletions(-)
include/linux/slab.h +16 -19
@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);* Kmalloc array related definitions */ -#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to -* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifdef CONFIG_SLAB +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW
@@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl* Try really hard to succeed the allocation but fail * eventually. */ +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) {if (__builtin_constant_p (size)) { -#ifndef CONFIG_SLOB unsigned int index; -#endif +if (size >KMALLOC_MAX_CACHE_SIZE )return kmalloc_large (size, flags); -#ifndef CONFIG_SLOB + index =kmalloc_index (size);if (!index)
@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)return kmem_cache_alloc_trace ( kmalloc_caches[kmalloc_type (flags)][index], flags, size); -#endif }return __kmalloc (size, flags); } +#else +static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags); +return __kmalloc (size, flags); +} +#endif static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) {
mm/slab.c +27 -4
@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags); + cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;
@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags); + cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;
@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) {struct kmem_cache *s;size_t i; +struct folio *folio;local_irq_disable ();for (i =0 ; i < size; i++) {void *objp = p[i]; -if (!orig_s)/* called via kfree_bulk */ +if (!orig_s) { +/* called via kfree_bulk */ + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, objp); +continue ; + } s =virt_to_cache (objp); -else + }else s =cache_from_obj (orig_s, objp);if (!s)continue ;
@@ -3776,11 +3785,20 @@ void kfree(const void *objp) {struct kmem_cache *c;unsigned long flags; +struct folio *folio; +void *object = (void *) objp;trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ; + + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, object); +return ; + } +local_irq_save (flags);kfree_debugcheck (objp); c =virt_to_cache (objp);
@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,size_t __ksize (const void *objp) {struct kmem_cache *c; +struct folio *folio;size_t size;BUG_ON (!objp);if (unlikely (objp ==ZERO_SIZE_PTR ))return 0 ; + folio =virt_to_folio (objp); +if (!folio_test_slab (folio)) +return folio_size (folio); + c =virt_to_cache (objp); size = c ? c->object_size :0 ;
mm/slab.h +19 -0
@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)print_tracking (cachep, x);return cachep; } + +static __always_inline void kfree_hook(void *x) +{ +kmemleak_free (x); +kasan_kfree_large (x); +} + +static inline void free_large_kmalloc (struct folio *folio,void *object) +{ +unsigned int order =folio_order (folio); + +if (WARN_ON_ONCE (order ==0 )) +pr_warn_once ("object pointer: 0x%p\n" , object); + +kfree_hook (object); +mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , + -(PAGE_SIZE << order)); +__free_pages (folio_page (folio,0 ), order); +}#endif /* CONFIG_SLOB */ static inline size_t slab_ksize (const struct kmem_cache *s)
mm/slub.c +0 -19
@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)return ptr; } -static __always_inline void kfree_hook(void *x) -{ -kmemleak_free (x); -kasan_kfree_large (x); -} - static __always_inline bool slab_free_hook(struct kmem_cache *s,void *x,bool init) {
@@ -3535,19 +3529,6 @@ struct detached_freelist {struct kmem_cache *s; }; -static inline void free_large_kmalloc (struct folio *folio,void *object) -{ -unsigned int order =folio_order (folio); - -if (WARN_ON_ONCE (order ==0 )) -pr_warn_once ("object pointer: 0x%p\n" , object); - -kfree_hook (object); -mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , - -(PAGE_SIZE << order)); -__free_pages (folio_page (folio,0 ), order); -} -/* * This function progressively scans the array with free objects (with * a limited look ahead) and extract objects belonging to the same
Signature · 1 line
2.33.1
Re: [PATCH v2 1/5] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
Quoted 5 lines · Hyeonggon Yoo
There is not much benefit for serving large objects in
kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.Signed-off-by Hyeonggon Yoo <[email protected]>
Looks like it's on the right path! Some suggestions follow.
Quoted 23 lines
--- include/linux/slab.h | 35 ++++++++++++++++------------------- mm/slab.c | 31 +++++++++++++++++++++++++++---- mm/slab.h | 19 +++++++++++++++++++ mm/slub.c | 19 ------------------- 4 files changed, 62 insertions(+), 42 deletions(-)include/linux/slab.h +0
-7@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);* Kmalloc array related definitions */ -#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to
include/linux/slab.h void kmem_dump_obj(void *object); · line 233
Quoted 10 lines
include/linux/slab.h +5
-4-* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifdef CONFIG_SLAB +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)
I think we should be able to also remove the larger sizes from __kmalloc_index() later in this file.
Quoted 56 lines
include/linux/slab.h +11
-8#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW @@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl* Try really hard to succeed the allocation but fail * eventually. */ +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) {if (__builtin_constant_p (size)) { -#ifndef CONFIG_SLOB unsigned int index; -#endif +if (size >KMALLOC_MAX_CACHE_SIZE )return kmalloc_large (size, flags); -#ifndef CONFIG_SLOB + index =kmalloc_index (size);if (!index)@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)return kmem_cache_alloc_trace ( kmalloc_caches[kmalloc_type (flags)][index], flags, size); -#endif }return __kmalloc (size, flags); } +#else +static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags); +return __kmalloc (size, flags); +} +#endif static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) {mm/slab.c +0
-0
mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3627
Quoted 5 lines
mm/slab.c +1
-1@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags);
This loses the node information and NUMA locality. It would be better to generalize kmalloc_large_node() from mm/slub.c instead.
Quoted 20 lines
mm/slab.c +4
-1+ cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags); + cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) {struct kmem_cache *s;size_t i; +struct folio *folio;
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3754
Quoted 10 lines
mm/slab.c +5
-1local_irq_disable ();for (i =0 ; i < size; i++) {void *objp = p[i]; -if (!orig_s)/* called via kfree_bulk */ +if (!orig_s) { +/* called via kfree_bulk */ + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, objp);
Hmm I'm not sure it's a good idea to call this and the page allocator with disabled irq's. Maybe simply re-enable around it?
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3757
Quoted 3 lines
mm/slab.c +2
-0+continue ; + } s =virt_to_cache (objp);
Since we already have the folio, we shoold instead use folio_slab and slab->slab_cache (see how mm/slub.c does it).
The same applies to functions below as well.
Quoted 49 lines
mm/slab.c +15
-1-else + }else s =cache_from_obj (orig_s, objp);if (!s)continue ;@@ -3776,11 +3785,20 @@ void kfree(const void *objp) {struct kmem_cache *c;unsigned long flags; +struct folio *folio; +void *object = (void *) objp;trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ; + + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, object); +return ; + } +local_irq_save (flags);kfree_debugcheck (objp); c =virt_to_cache (objp);@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,size_t __ksize (const void *objp) {struct kmem_cache *c; +struct folio *folio;size_t size;BUG_ON (!objp);if (unlikely (objp ==ZERO_SIZE_PTR ))return 0 ; + folio =virt_to_folio (objp); +if (!folio_test_slab (folio)) +return folio_size (folio); + c =virt_to_cache (objp); size = c ? c->object_size :0 ;mm/slab.h +0
-0
mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 671
Quoted 8 lines
mm/slab.h +5
-0@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)print_tracking (cachep, x);return cachep; } + +static __always_inline void kfree_hook(void *x) +{ +kmemleak_free (x); +kasan_kfree_large (x);
Looks like there's only one caller of kfree_hook() so we could do that directly there.
Quoted 54 lines
mm/slab.h +14
-0+} + +static inline void free_large_kmalloc (struct folio *folio,void *object) +{ +unsigned int order =folio_order (folio); + +if (WARN_ON_ONCE (order ==0 )) +pr_warn_once ("object pointer: 0x%p\n" , object); + +kfree_hook (object); +mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , + -(PAGE_SIZE << order)); +__free_pages (folio_page (folio,0 ), order); +}#endif /* CONFIG_SLOB */ static inline size_t slab_ksize (const struct kmem_cache *s)mm/slub.c +0
-19@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)return ptr; } -static __always_inline void kfree_hook(void *x) -{ -kmemleak_free (x); -kasan_kfree_large (x); -} - static __always_inline bool slab_free_hook(struct kmem_cache *s,void *x,bool init) {@@ -3535,19 +3529,6 @@ struct detached_freelist {struct kmem_cache *s; }; -static inline void free_large_kmalloc (struct folio *folio,void *object) -{ -unsigned int order =folio_order (folio); - -if (WARN_ON_ONCE (order ==0 )) -pr_warn_once ("object pointer: 0x%p\n" , object); - -kfree_hook (object); -mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , - -(PAGE_SIZE << order)); -__free_pages (folio_page (folio,0 ), order); -} -/* * This function progressively scans the array with free objects (with * a limited look ahead) and extract objects belonging to the same
Re: [PATCH v2 1/5] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator
Quoted 9 lines · Vlastimil Babka
Quoted 5 lines · Hyeonggon Yoo
There is not much benefit for serving large objects in
kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.Signed-off-by Hyeonggon Yoo <[email protected]>Looks like it's on the right path! Some suggestions follow.
Thank you for reviewing this!
You have really good eye.
Quoted 28 lines
Quoted 28 lines
--- include/linux/slab.h | 35 ++++++++++++++++------------------- mm/slab.c | 31 +++++++++++++++++++++++++++---- mm/slab.h | 19 +++++++++++++++++++ mm/slub.c | 19 ------------------- 4 files changed, 62 insertions(+), 42 deletions(-)include/linux/slab.h +2
-9@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);* Kmalloc array related definitions */ -#ifdef CONFIG_SLAB /* -* The largest kmalloc size supported by the SLAB allocators is -* 32 megabyte (2^25) or the maximum allocatable page order if that is -* less than 32 MB. -* -* WARNING: Its not easy to increase this value since the allocators have -* to do various tricks to work around compiler limitations in order to -* ensure proper constant folding. +* SLAB and SLUB directly allocates requests fitting in to an order-1 page +* (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
Quoted 9 lines
include/linux/slab.h void kmem_dump_obj(void *object); · line 233
Quoted 5 lines
include/linux/slab.h +3
-2- (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifdef CONFIG_SLAB +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)I think we should be able to also remove the larger sizes from
__kmalloc_index()later in this file.
Right. we can remove sizes larger than max(PAGE_SHIFT) + 1.
searching using cscope, maximum PAGE_SHIFT is 20 (1MB) in hexagon. I think we can remove anything larger than 2MB.
But Hmm, I'll add something like static_assert(PAGE_SHIFT <= 20). Let's see if any architecture/config complain this.
Quoted 58 lines
Quoted 58 lines
include/linux/slab.h +11
-8#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5#endif #endif #ifdef CONFIG_SLUB -/* -* SLUB directly allocates requests fitting in to an order-1 page -* (PAGE_SIZE*2). Larger requests are passed to the page allocator. -*/ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)#ifndef KMALLOC_SHIFT_LOW @@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl* Try really hard to succeed the allocation but fail * eventually. */ +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) {if (__builtin_constant_p (size)) { -#ifndef CONFIG_SLOB unsigned int index; -#endif +if (size >KMALLOC_MAX_CACHE_SIZE )return kmalloc_large (size, flags); -#ifndef CONFIG_SLOB + index =kmalloc_index (size);if (!index)@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)return kmem_cache_alloc_trace ( kmalloc_caches[kmalloc_type (flags)][index], flags, size); -#endif }return __kmalloc (size, flags); } +#else +static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) +{ +if (__builtin_constant_p (size) && size >KMALLOC_MAX_CACHE_SIZE ) +return kmalloc_large (size, flags); +return __kmalloc (size, flags); +} +#endif static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) {mm/slab.c +0
-0@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *ret;
Quoted 5 lines
mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3627
Quoted 3 lines
mm/slab.c +1
-1if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags);This loses the node information and NUMA locality.
Oh, my fault. Thank you for catching this.
Quoted 1 line
It would be better to generalize
kmalloc_large_node()frommm/slub.cinstead.
Sounds nice.
Quoted 24 lines
Quoted 24 lines
mm/slab.c +4
-1+ cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,void *ret;if (unlikely (size >KMALLOC_MAX_CACHE_SIZE )) -return NULL ; +return kmalloc_large (size, flags); + cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep;@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) {struct kmem_cache *s;size_t i; +struct folio *folio;local_irq_disable ();for (i =0 ; i < size; i++) {void *objp = p[i];
Quoted 9 lines
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3754
Quoted 6 lines
mm/slab.c +5
-1-if (!orig_s)/* called via kfree_bulk */ +if (!orig_s) { +/* called via kfree_bulk */ + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, objp);Hmm I'm not sure it's a good idea to call this and the page allocator with disabled irq's. Maybe simply re-enable around it?
Seems reasonable. re-enabling around here will be sufficient for this function.
By The Way, it seems SLAB really does not care about disabling irq even if it's unnecessary.
I would like to try reducing irq-disabled area in SLAB and how it affects on macro/micro benchmarks after this series is done :)
Quoted 7 lines
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3757
Quoted 3 lines
mm/slab.c +2
-0+continue ; + } s =virt_to_cache (objp);Since we already have the folio, we shoold instead use
folio_slabandslab->slab_cache(see howmm/slub.cdoes it).
The same applies to functions below as well.
Right. I'll change them in v3.
Quoted 44 lines
Quoted 44 lines
mm/slab.c +15
-1-else + }else s =cache_from_obj (orig_s, objp);if (!s)continue ;@@ -3776,11 +3785,20 @@ void kfree(const void *objp) {struct kmem_cache *c;unsigned long flags; +struct folio *folio; +void *object = (void *) objp;trace_kfree (_RET_IP_, objp);if (unlikely (ZERO_OR_NULL_PTR (objp)))return ; + + folio =virt_to_folio (objp); +if (unlikely (!folio_test_slab (folio))) { +free_large_kmalloc (folio, object); +return ; + } +local_irq_save (flags);kfree_debugcheck (objp); c =virt_to_cache (objp);@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,size_t __ksize (const void *objp) {struct kmem_cache *c; +struct folio *folio;size_t size;BUG_ON (!objp);if (unlikely (objp ==ZERO_SIZE_PTR ))return 0 ; + folio =virt_to_folio (objp); +if (!folio_test_slab (folio)) +return folio_size (folio); + c =virt_to_cache (objp); size = c ? c->object_size :0 ;
Quoted 17 lines
mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 671
Quoted 13 lines
mm/slab.h +5
-0@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)print_tracking (cachep, x);return cachep; } + +static __always_inline void kfree_hook(void *x) +{ +kmemleak_free (x); +kasan_kfree_large (x);Looks like there's only one caller of
kfree_hook()so we could do that directly there.
Right.
Quoted 38 lines
Quoted 38 lines
mm/slab.h +14
-0+} + +static inline void free_large_kmalloc (struct folio *folio,void *object) +{ +unsigned int order =folio_order (folio); + +if (WARN_ON_ONCE (order ==0 )) +pr_warn_once ("object pointer: 0x%p\n" , object); + +kfree_hook (object); +mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , + -(PAGE_SIZE << order)); +__free_pages (folio_page (folio,0 ), order); +}#endif /* CONFIG_SLOB */ static inline size_t slab_ksize (const struct kmem_cache *s)mm/slub.c +0
-6@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)return ptr; } -static __always_inline void kfree_hook(void *x) -{ -kmemleak_free (x); -kasan_kfree_large (x); -} - static __always_inline bool slab_free_hook(struct kmem_cache *s,void *x,bool init) {@@ -3535,19 +3529,6 @@ struct detached_freelist {struct kmem_cache *s; };
Quoted 17 lines
Quoted 6 lines
mm/slub.c +0
-6-static inline void free_large_kmalloc (struct folio *folio,void *object) -{ -unsigned int order =folio_order (folio); - -if (WARN_ON_ONCE (order ==0 )) -pr_warn_once ("object pointer: 0x%p\n" , object);mm/slub.c struct detached_freelist { · line 3534
Quoted 10 lines
mm/slub.c +0
-7- -kfree_hook (object); -mod_lruvec_page_state (folio_page (folio,0 ),NR_SLAB_UNRECLAIMABLE_B , - -(PAGE_SIZE << order)); -__free_pages (folio_page (folio,0 ), order); -} -/* * This function progressively scans the array with free objects (with * a limited look ahead) and extract objects belonging to the same
Thank you so much!
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)
PATCH v2 2/5 mm/sl[au]b: unify __ksize()
In reply to Hyeonggon YooNow that SLAB passes large requests to page allocator like SLUB, Unify __ksize(). Only SLOB need to implement own version of __ksize() because it stores size in object header for kmalloc objects.
--- mm/slab.c | 30 ------------------------------ mm/slab_common.c | 27 +++++++++++++++++++++++++++ mm/slub.c | 16 ---------------- 3 files changed, 27 insertions(+), 46 deletions(-)
mm/slab.c +0 -30
@@ -4216,33 +4216,3 @@ void __check_heap_object(const void *ptr, unsigned long n, usercopy_abort("SLAB object", cachep->name, to_user, offset, n); }#endif /* CONFIG_HARDENED_USERCOPY */ - -/** -* __ksize -- Uninstrumented ksize. -* @objp: pointer to the object -* -* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same -* safety checks as ksize() with KASAN instrumentation enabled. -* -* Return: size of the actual memory used by @objp in bytes -*/ -size_t __ksize (const void *objp) -{ -struct kmem_cache *c; -struct folio *folio; -size_t size; - -BUG_ON (!objp); -if (unlikely (objp ==ZERO_SIZE_PTR )) -return 0 ; - - folio =virt_to_folio (objp); -if (!folio_test_slab (folio)) -return folio_size (folio); - - c =virt_to_cache (objp); - size = c ? c->object_size :0 ; - -return size; -} -EXPORT_SYMBOL (__ksize);
mm/slab_common.c +27 -0
@@ -1245,6 +1245,33 @@ void kfree_sensitive(const void *p) }EXPORT_SYMBOL (kfree_sensitive); +#ifndef CONFIG_SLOB +/** +* __ksize -- Uninstrumented ksize. +* @objp: pointer to the object +* +* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same +* safety checks as ksize() with KASAN instrumentation enabled. +* +* Return: size of the actual memory used by @objp in bytes +*/ +size_t __ksize (const void *object) +{ +struct folio *folio; + +if (unlikely (object ==ZERO_SIZE_PTR )) +return 0 ; + + folio =virt_to_folio (object); + +if (unlikely (!folio_test_slab (folio))) +return folio_size (folio); + +return slab_ksize (folio_slab (folio)->slab_cache); +} +EXPORT_SYMBOL (__ksize); +#endif +/** * ksize - get the actual amount of memory allocated for a given object * @objp: Pointer to the object
mm/slub.c +0 -16
@@ -4507,22 +4507,6 @@ void __check_heap_object(const void *ptr, unsigned long n, }#endif /* CONFIG_HARDENED_USERCOPY */ -size_t __ksize (const void *object) -{ -struct folio *folio; - -if (unlikely (object ==ZERO_SIZE_PTR )) -return 0 ; - - folio =virt_to_folio (object); - -if (unlikely (!folio_test_slab (folio))) -return folio_size (folio); - -return slab_ksize (folio_slab (folio)->slab_cache); -} -EXPORT_SYMBOL (__ksize); -void kfree (const void *x) {struct folio *folio;
Signature · 1 line
2.33.1
Re: [PATCH v2 2/5] mm/sl[au]b: unify __ksize()
Quoted 5 lines · Hyeonggon Yoo
Now that SLAB passes large requests to page allocator like SLUB, Unify
__ksize(). Only SLOB need to implement own version of__ksize()because it stores size in object header for kmalloc objects.Signed-off-by Hyeonggon Yoo <[email protected]>
As discussed, we can be more specific about the !folio_test_slab() case, but that can be done on top.
Re: [PATCH v2 2/5] mm/sl[au]b: unify __ksize()
Quoted 11 lines · Vlastimil Babka
Quoted 5 lines · Hyeonggon Yoo
Now that SLAB passes large requests to page allocator like SLUB, Unify
__ksize(). Only SLOB need to implement own version of__ksize()because it stores size in object header for kmalloc objects.Signed-off-by Hyeonggon Yoo <[email protected]>Reviewed-by Vlastimil Babka <[email protected]>As discussed, we can be more specific about the !folio_test_slab() case, but that can be done on top.
I'll add that in v3. Thanks!
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)
PATCH v2 3/5 mm/sl[auo]b: move definition of __ksize() to mm/slab.h
In reply to Hyeonggon Yoo__ksize() is only called by KASAN. Remove export symbol and move definition to mm/slab.h as we don't want to grow its callers.
[ [email protected]: Move definition to mm/slab.h and reduce comments ]
--- include/linux/slab.h | 1 - mm/slab.h | 2 ++ mm/slab_common.c | 9 +-------- mm/slob.c | 1 - 4 files changed, 3 insertions(+), 10 deletions(-)
include/linux/slab.h +0 -1
@@ -182,7 +182,6 @@ int kmem_cache_shrink(struct kmem_cache *s);void *__must_check krealloc (const void *objp,size_t new_size,gfp_t flags)__alloc_size(2) ;void kfree (const void *objp);void kfree_sensitive (const void *objp); -size_t __ksize (const void *objp);size_t ksize (const void *objp);#ifdef CONFIG_PRINTK bool kmem_valid_obj (void *object);
mm/slab.h +2 -0
@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object) }#endif /* CONFIG_SLOB */ +size_t __ksize (const void *objp); +static inline size_t slab_ksize (const struct kmem_cache *s) {#ifndef CONFIG_SLUB
mm/slab_common.c +1 -8
@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);#ifndef CONFIG_SLOB /** -* __ksize -- Uninstrumented ksize. -* @objp: pointer to the object -* -* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same -* safety checks as ksize() with KASAN instrumentation enabled. -* -* Return: size of the actual memory used by @objp in bytes +* __ksize -- Uninstrumented ksize. Only called by KASAN. */ size_t __ksize (const void *object) {
@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object) return slab_ksize(folio_slab(folio)->slab_cache); } -EXPORT_SYMBOL (__ksize);#endif /**
mm/slob.c +0 -1
@@ -588,7 +588,6 @@ size_t __ksize(const void *block) m = (unsigned int *)(block - align);return SLOB_UNITS (*m) *SLOB_UNIT ; } -EXPORT_SYMBOL (__ksize);int __kmem_cache_create (struct kmem_cache *c,slab_flags_t flags) {
Signature · 1 line
2.33.1
Re: [PATCH v2 3/5] mm/sl[auo]b: move definition of __ksize() to mm/slab.h
Quoted 6 lines · Hyeonggon Yoo
__ksize()is only called by KASAN. Remove export symbol and move definition tomm/slab.has we don't want to grow its callers.[ [email protected]: Move definition to
mm/slab.hand reduce comments ]Signed-off-by Hyeonggon Yoo <[email protected]>
Quoted 16 lines
mm/slab.h +2
-0@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object) }#endif /* CONFIG_SLOB */ +size_t __ksize (const void *objp); +static inline size_t slab_ksize (const struct kmem_cache *s) {#ifndef CONFIG_SLUB mm/slab_common.c +0
-0
mm/slab_common.c EXPORT_SYMBOL(kfree_sensitive); · line 1249
Quoted 3 lines
mm/slab_common.c +0
-0@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);#ifndef CONFIG_SLOB /**
Maybe just /* so it's not even parsed as a kernel-doc anymore?
Quoted 18 lines
mm/slab_common.c +1
-8-* __ksize -- Uninstrumented ksize. -* @objp: pointer to the object -* -* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same -* safety checks as ksize() with KASAN instrumentation enabled. -* -* Return: size of the actual memory used by @objp in bytes +* __ksize -- Uninstrumented ksize. Only called by KASAN. */ size_t __ksize (const void *object) {@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object) return slab_ksize(folio_slab(folio)->slab_cache); } -EXPORT_SYMBOL (__ksize);#endif
Re: [PATCH v2 3/5] mm/sl[auo]b: move definition of __ksize() to mm/slab.h
Quoted 10 lines · Vlastimil Babka
Quoted 6 lines · Hyeonggon Yoo
__ksize()is only called by KASAN. Remove export symbol and move definition tomm/slab.has we don't want to grow its callers.[ [email protected]: Move definition to
mm/slab.hand reduce comments ]Signed-off-by Hyeonggon Yoo <[email protected]>Reviewed-by Vlastimil Babka <[email protected]>
Thanks!
Quoted 17 lines
Quoted 17 lines
mm/slab.h +2
-0@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object) }#endif /* CONFIG_SLOB */ +size_t __ksize (const void *objp); +static inline size_t slab_ksize (const struct kmem_cache *s) {#ifndef CONFIG_SLUB mm/slab_common.c +0
-0@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);
Quoted 5 lines
mm/slab_common.c EXPORT_SYMBOL(kfree_sensitive); · line 1249
Quoted 2 lines
mm/slab_common.c +0
-0#ifndef CONFIG_SLOB /** Maybe just /* so it's not even parsed as a kernel-doc anymore?
Oh yes, that would be better.
Quoted 18 lines
Quoted 18 lines
mm/slab_common.c +1
-8-* __ksize -- Uninstrumented ksize. -* @objp: pointer to the object -* -* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same -* safety checks as ksize() with KASAN instrumentation enabled. -* -* Return: size of the actual memory used by @objp in bytes +* __ksize -- Uninstrumented ksize. Only called by KASAN. */ size_t __ksize (const void *object) {@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object) return slab_ksize(folio_slab(folio)->slab_cache); } -EXPORT_SYMBOL (__ksize);#endif
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)
PATCH v2 4/5 mm/slub: limit number of node partial slabs only in cache creation
In reply to Hyeonggon YooSLUB sets number of minimum partial slabs for node (min_partial) using set_min_partial(). SLUB holds at least min_partial slabs even if they're empty to avoid excessive use of page allocator.
set_min_partial() limits value of min_partial limits value of min_partial MIN_PARTIAL and MAX_PARTIAL. As set_min_partial() can be called by min_partial_store() too, Only limit value of min_partial in kmem_cache_open() so that it can be changed to value that a user wants.
[ [email protected]: Fold set_min_partial() into its callers ]
--- mm/slub.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-)
mm/slub.c +3 -11
@@ -3981,15 +3981,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)return 1 ; } -static void set_min_partial (struct kmem_cache *s,unsigned long min) -{ -if (min <MIN_PARTIAL ) - min =MIN_PARTIAL ; -else if (min >MAX_PARTIAL ) - min =MAX_PARTIAL ; - s->min_partial = min; -} -static void set_cpu_partial (struct kmem_cache *s) {#ifdef CONFIG_SLUB_CPU_PARTIAL
@@ -4196,7 +4187,8 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)* The larger the object size is, the more slabs we want on the partial * list to avoid pounding the page allocator excessively. */ -set_min_partial (s,ilog2 (s->size) /2 ); + s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); + s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);set_cpu_partial (s);
@@ -5361,7 +5353,7 @@ static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,if (err)return err; -set_min_partial (s, min); + s->min_partial = min;return length; }SLAB_ATTR (min_partial);
Signature · 1 line
2.33.1
Re: [PATCH v2 4/5] mm/slub: limit number of node partial slabs only in cache creation
Quoted 4 lines · Hyeonggon Yoo
SLUB sets number of minimum partial slabs for node (
min_partial) usingset_min_partial(). SLUB holds at leastmin_partialslabs even if they're empty to avoid excessive use of page allocator.
Quoted 8 lines
set_min_partial()limits value ofmin_partiallimits value ofmin_partialMIN_PARTIALandMAX_PARTIAL. Asset_min_partial()can be called bymin_partial_store()too, Only limit value ofmin_partialinkmem_cache_open()so that it can be changed to value that a user wants.[ [email protected]: Fold
set_min_partial()into its callers ]Signed-off-by Hyeonggon Yoo <[email protected]>
Quoted 43 lines
--- mm/slub.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-)mm/slub.c +3
-11@@ -3981,15 +3981,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)return 1 ; } -static void set_min_partial (struct kmem_cache *s,unsigned long min) -{ -if (min <MIN_PARTIAL ) - min =MIN_PARTIAL ; -else if (min >MAX_PARTIAL ) - min =MAX_PARTIAL ; - s->min_partial = min; -} -static void set_cpu_partial (struct kmem_cache *s) {#ifdef CONFIG_SLUB_CPU_PARTIAL @@ -4196,7 +4187,8 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)* The larger the object size is, the more slabs we want on the partial * list to avoid pounding the page allocator excessively. */ -set_min_partial (s,ilog2 (s->size) /2 ); + s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); + s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);set_cpu_partial (s);@@ -5361,7 +5353,7 @@ static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,if (err)return err; -set_min_partial (s, min); + s->min_partial = min;return length; }SLAB_ATTR (min_partial);
PATCH v2 5/5 mm/slub: refactor deactivate_slab()
In reply to Hyeonggon YooSimplify deactivate_slab() by unlocking n->list_lock and retrying cmpxchg_double() when cmpxchg_double() fails, and perform add_{partial,full} only when it succeed.
Releasing and taking n->list_lock again here is not harmful as SLUB avoids deactivating slabs as much as possible.
[ [email protected]: perform add_{partial,full} when cmpxchg_double()
succeed. ]
--- mm/slub.c | 81 ++++++++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 49 deletions(-)
mm/slub.c +32 -49
@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, {enum slab_modes {M_NONE ,M_PARTIAL ,M_FULL ,M_FREE };struct kmem_cache_node *n =get_node (s,slab_nid (slab)); -int lock =0 , free_delta =0 ; -enum slab_modes l =M_NONE , m =M_NONE ; +int free_delta =0 ; +enum slab_modes mode =M_NONE ;void *nextfree, *freelist_iter, *freelist_tail;int tail =DEACTIVATE_TO_HEAD ;unsigned long flags =0 ;
@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,* Ensure that the slab is unfrozen while the list presence * reflects the actual number of objects during unfreeze. * -* We setup the list membership and then perform a cmpxchg -* with the count. If there is a mismatch then the slab -* is not unfrozen but the slab is on the wrong list. -* -* Then we restart the process which may have to remove -* the slab from the list that we just put it on again -* because the number of objects in the slab may have -* changed. +* We first perform cmpxchg holding lock and insert to list +* when it succeed. If there is mismatch then slub is not +* unfrozen and number of objects in the slab may have changed. +* Then release lock and retry cmpxchg again. */ redo:
@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, new.frozen =0 ;if (!new.inuse && n->nr_partial >= s->min_partial) - m =M_FREE ; + mode =M_FREE ;else if (new.freelist) { - m =M_PARTIAL ; -if (!lock) { - lock =1 ; -/* -* Taking the spinlock removes the possibility that -* acquire_slab() will see a slab that is frozen -*/ -spin_lock_irqsave (&n->list_lock, flags); - } - }else { - m =M_FULL ; -if (kmem_cache_debug_flags (s,SLAB_STORE_USER ) && !lock) { - lock =1 ; -/* -* This also ensures that the scanning of full -* slabs from diagnostic functions will not see -* any frozen slabs. -*/ -spin_lock_irqsave (&n->list_lock, flags); - } + mode =M_PARTIAL ; +/* +* Taking the spinlock removes the possibility that +* acquire_slab() will see a slab that is frozen +*/ +spin_lock_irqsave (&n->list_lock, flags); + }else if (kmem_cache_debug_flags (s,SLAB_STORE_USER )) { + mode =M_FULL ; +/* +* This also ensures that the scanning of full +* slabs from diagnostic functions will not see +* any frozen slabs. +*/ +spin_lock_irqsave (&n->list_lock, flags); } -if (l != m) { -if (l ==M_PARTIAL ) -remove_partial (n, slab); -else if (l ==M_FULL ) -remove_full (s, n, slab); - -if (m ==M_PARTIAL ) -add_partial (n, slab, tail); -else if (m ==M_FULL ) -add_full (s, n, slab); - } - l = m;if (!cmpxchg_double_slab (s, slab, old.freelist, old.counters, new.freelist, new.counters, -"unfreezing slab" )) +"unfreezing slab" )) { +if (mode ==M_PARTIAL || mode ==M_FULL ) +spin_unlock_irqrestore (&n->list_lock, flags); goto redo; + } -if (lock) -spin_unlock_irqrestore (&n->list_lock, flags); -if (m ==M_PARTIAL ) +if (mode ==M_PARTIAL ) { +add_partial (n, slab, tail); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s, tail); -else if (m ==M_FULL ) + }else if (mode ==M_FULL ) { +add_full (s, n, slab); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s,DEACTIVATE_FULL ); -else if (m ==M_FREE ) { + }else if (mode ==M_FREE ) {stat (s,DEACTIVATE_EMPTY );discard_slab (s, slab);stat (s,FREE_SLAB );
Signature · 1 line
2.33.1
Re: [PATCH v2 5/5] mm/slub: refactor deactivate_slab()
Quoted 11 lines · Hyeonggon Yoo
Simplify
deactivate_slab()by unlockingn->list_lockand retryingcmpxchg_double()whencmpxchg_double()fails, and perform add_{partial,full} only when it succeed.Releasing and taking
n->list_lockagain here is not harmful as SLUB avoids deactivating slabs as much as possible.[ [email protected]: perform add_{partial,full} when
cmpxchg_double()
succeed. ]Signed-off-by Hyeonggon Yoo <[email protected]>
Looks good, just noticed a tiny issue.
Quoted 47 lines
--- mm/slub.c | 81 ++++++++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 49 deletions(-)mm/slub.c +7
-13@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, {enum slab_modes {M_NONE ,M_PARTIAL ,M_FULL ,M_FREE };struct kmem_cache_node *n =get_node (s,slab_nid (slab)); -int lock =0 , free_delta =0 ; -enum slab_modes l =M_NONE , m =M_NONE ; +int free_delta =0 ; +enum slab_modes mode =M_NONE ;void *nextfree, *freelist_iter, *freelist_tail;int tail =DEACTIVATE_TO_HEAD ;unsigned long flags =0 ;@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,* Ensure that the slab is unfrozen while the list presence * reflects the actual number of objects during unfreeze. * -* We setup the list membership and then perform a cmpxchg -* with the count. If there is a mismatch then the slab -* is not unfrozen but the slab is on the wrong list. -* -* Then we restart the process which may have to remove -* the slab from the list that we just put it on again -* because the number of objects in the slab may have -* changed. +* We first perform cmpxchg holding lock and insert to list +* when it succeed. If there is mismatch then slub is not +* unfrozen and number of objects in the slab may have changed. +* Then release lock and retry cmpxchg again. */ redo:@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, new.frozen =0 ;if (!new.inuse && n->nr_partial >= s->min_partial) - m =M_FREE ; + mode =M_FREE ;else if (new.freelist) { - m =M_PARTIAL ; -if (!lock) {
mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2430
Quoted 10 lines
mm/slub.c +0
-10- lock =1 ; -/* -* Taking the spinlock removes the possibility that -* acquire_slab() will see a slab that is frozen -*/ -spin_lock_irqsave (&n->list_lock, flags); - } - }else { - m =M_FULL ; -if (kmem_cache_debug_flags (s,SLAB_STORE_USER ) && !lock) {
This used to set m = M_FULL; always.
Quoted 6 lines
mm/slub.c +0
-6- lock =1 ; -/* -* This also ensures that the scanning of full -* slabs from diagnostic functions will not see -* any frozen slabs. -*/
mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2422
Quoted 10 lines
mm/slub.c +8
-2-spin_lock_irqsave (&n->list_lock, flags); - } + mode =M_PARTIAL ; +/* +* Taking the spinlock removes the possibility that +* acquire_slab() will see a slab that is frozen +*/ +spin_lock_irqsave (&n->list_lock, flags); + }else if (kmem_cache_debug_flags (s,SLAB_STORE_USER )) { + mode =M_FULL ;
Now you only set it for SLAB_STORE_USER caches.
Quoted 34 lines
mm/slub.c +10
-15+/* +* This also ensures that the scanning of full +* slabs from diagnostic functions will not see +* any frozen slabs. +*/ +spin_lock_irqsave (&n->list_lock, flags); } -if (l != m) { -if (l ==M_PARTIAL ) -remove_partial (n, slab); -else if (l ==M_FULL ) -remove_full (s, n, slab); - -if (m ==M_PARTIAL ) -add_partial (n, slab, tail); -else if (m ==M_FULL ) -add_full (s, n, slab); - } - l = m;if (!cmpxchg_double_slab (s, slab, old.freelist, old.counters, new.freelist, new.counters, -"unfreezing slab" )) +"unfreezing slab" )) { +if (mode ==M_PARTIAL || mode ==M_FULL ) +spin_unlock_irqrestore (&n->list_lock, flags); goto redo; + } -if (lock) -spin_unlock_irqrestore (&n->list_lock, flags);
mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2449
Quoted 10 lines
mm/slub.c +6
-2-if (m ==M_PARTIAL ) +if (mode ==M_PARTIAL ) { +add_partial (n, slab, tail); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s, tail); -else if (m ==M_FULL ) + }else if (mode ==M_FULL ) { +add_full (s, n, slab); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s,DEACTIVATE_FULL );
As a result, full slabs without SLAB_STORE_USER will not count DEACTIVATE_FULL anymore.
I guess the easiest way to solve it is to e.g. add a M_FULL_NOLIST mode that only does the DEACTIVATE_FULL counting.
Quoted 5 lines
mm/slub.c +1
-1-else if (m ==M_FREE ) { + }else if (mode ==M_FREE ) {stat (s,DEACTIVATE_EMPTY );discard_slab (s, slab);stat (s,FREE_SLAB );
Re: [PATCH v2 5/5] mm/slub: refactor deactivate_slab()
Quoted 128 lines · Vlastimil Babka
Quoted 11 lines · Hyeonggon Yoo
Simplify
deactivate_slab()by unlockingn->list_lockand retryingcmpxchg_double()whencmpxchg_double()fails, and perform add_{partial,full} only when it succeed.Releasing and taking
n->list_lockagain here is not harmful as SLUB avoids deactivating slabs as much as possible.[ [email protected]: perform add_{partial,full} when
cmpxchg_double()
succeed. ]Signed-off-by Hyeonggon Yoo <[email protected]>Looks good, just noticed a tiny issue.
Quoted 47 lines
--- mm/slub.c | 81 ++++++++++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 49 deletions(-)mm/slub.c +7
-13@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, {enum slab_modes {M_NONE ,M_PARTIAL ,M_FULL ,M_FREE };struct kmem_cache_node *n =get_node (s,slab_nid (slab)); -int lock =0 , free_delta =0 ; -enum slab_modes l =M_NONE , m =M_NONE ; +int free_delta =0 ; +enum slab_modes mode =M_NONE ;void *nextfree, *freelist_iter, *freelist_tail;int tail =DEACTIVATE_TO_HEAD ;unsigned long flags =0 ;@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,* Ensure that the slab is unfrozen while the list presence * reflects the actual number of objects during unfreeze. * -* We setup the list membership and then perform a cmpxchg -* with the count. If there is a mismatch then the slab -* is not unfrozen but the slab is on the wrong list. -* -* Then we restart the process which may have to remove -* the slab from the list that we just put it on again -* because the number of objects in the slab may have -* changed. +* We first perform cmpxchg holding lock and insert to list +* when it succeed. If there is mismatch then slub is not +* unfrozen and number of objects in the slab may have changed. +* Then release lock and retry cmpxchg again. */ redo:@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, new.frozen =0 ;if (!new.inuse && n->nr_partial >= s->min_partial) - m =M_FREE ; + mode =M_FREE ;else if (new.freelist) { - m =M_PARTIAL ; -if (!lock) {mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2430
Quoted 10 lines
mm/slub.c +0
-10- lock =1 ; -/* -* Taking the spinlock removes the possibility that -* acquire_slab() will see a slab that is frozen -*/ -spin_lock_irqsave (&n->list_lock, flags); - } - }else { - m =M_FULL ; -if (kmem_cache_debug_flags (s,SLAB_STORE_USER ) && !lock) {This used to set m =
M_FULL; always.Quoted 6 lines
mm/slub.c +0
-6- lock =1 ; -/* -* This also ensures that the scanning of full -* slabs from diagnostic functions will not see -* any frozen slabs. -*/ mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2422
Quoted 10 lines
mm/slub.c +8
-2-spin_lock_irqsave (&n->list_lock, flags); - } + mode =M_PARTIAL ; +/* +* Taking the spinlock removes the possibility that +* acquire_slab() will see a slab that is frozen +*/ +spin_lock_irqsave (&n->list_lock, flags); + }else if (kmem_cache_debug_flags (s,SLAB_STORE_USER )) { + mode =M_FULL ;Now you only set it for
SLAB_STORE_USERcaches.Quoted 34 lines
mm/slub.c +10
-15+/* +* This also ensures that the scanning of full +* slabs from diagnostic functions will not see +* any frozen slabs. +*/ +spin_lock_irqsave (&n->list_lock, flags); } -if (l != m) { -if (l ==M_PARTIAL ) -remove_partial (n, slab); -else if (l ==M_FULL ) -remove_full (s, n, slab); - -if (m ==M_PARTIAL ) -add_partial (n, slab, tail); -else if (m ==M_FULL ) -add_full (s, n, slab); - } - l = m;if (!cmpxchg_double_slab (s, slab, old.freelist, old.counters, new.freelist, new.counters, -"unfreezing slab" )) +"unfreezing slab" )) { +if (mode ==M_PARTIAL || mode ==M_FULL ) +spin_unlock_irqrestore (&n->list_lock, flags); goto redo; + } -if (lock) -spin_unlock_irqrestore (&n->list_lock, flags);
Quoted 13 lines
mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2449
Quoted 10 lines
mm/slub.c +6
-2-if (m ==M_PARTIAL ) +if (mode ==M_PARTIAL ) { +add_partial (n, slab, tail); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s, tail); -else if (m ==M_FULL ) + }else if (mode ==M_FULL ) { +add_full (s, n, slab); +spin_unlock_irqrestore (&n->list_lock, flags);stat (s,DEACTIVATE_FULL );As a result, full slabs without
SLAB_STORE_USERwill not countDEACTIVATE_FULLanymore.
Oh, thank you for catching this.
We usually only deactivate full slabs for debugging to track all list of slabs.
But as you pointed, in rare case when pfmemalloc flag does not match, full slabs can be deactivated (even if they are not put on list).
Quoted 2 lines
I guess the easiest way to solve it is to e.g. add a
M_FULL_NOLISTmode that only does theDEACTIVATE_FULLcounting.
That will be enough solution.
Will fix this in v3. Thank you!
Quoted 6 lines
Quoted 5 lines
mm/slub.c +1
-1-else if (m ==M_FREE ) { + }else if (mode ==M_FREE ) {stat (s,DEACTIVATE_EMPTY );discard_slab (s, slab);stat (s,FREE_SLAB );
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)
Quoted 11 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.
Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]
Quoted 25 lines
Hello, these are cleanup patches for slab.
Please consider them for slab-next :)Any comments will be appreciated.
Thanks.Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify__ksize()
mm/sl[auo]b: move definition of__ksize()tomm/slab.hmm/slub: limit number of node partial slabs only in cache creation mm/slub: refactordeactivate_slab()include/linux/slab.h | 36 ++++++------ mm/slab.c | 51 ++++++++--------- mm/slab.h | 21 +++++++ mm/slab_common.c | 20 +++++++ mm/slob.c | 1 - mm/slub.c | 130 ++++++++++++------------------------------- 6 files changed, 114 insertions(+), 145 deletions(-)--
2.33.1
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
Quoted 7 lines
Quoted 3 lines
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]
Oh sorry, I didn't understand why 0 was safer when I was reading it.
Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?
Quoted 25 lines
Quoted 25 lines
Hello, these are cleanup patches for slab.
Please consider them for slab-next :)Any comments will be appreciated.
Thanks.Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify__ksize()
mm/sl[auo]b: move definition of__ksize()tomm/slab.hmm/slub: limit number of node partial slabs only in cache creation mm/slub: refactordeactivate_slab()include/linux/slab.h | 36 ++++++------ mm/slab.c | 51 ++++++++--------- mm/slab.h | 21 +++++++ mm/slab_common.c | 20 +++++++ mm/slob.c | 1 - mm/slub.c | 130 ++++++++++++------------------------------- 6 files changed, 114 insertions(+), 145 deletions(-)--
2.33.1
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)
Quoted 15 lines · Hyeonggon Yoo
Quoted 13 lines · Marco Elver
Quoted 11 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.
Quoted 8 lines
Quoted 3 lines
Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]Oh sorry, I didn't understand why 0 was safer when I was reading it.
Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?
Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0. The intuition here is that the caller has a pointer to an invalid object, and wants to use ksize() to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by having ksize() return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj); touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.
The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)
Quoted 22 lines · Marco Elver
Quoted 21 lines · Hyeonggon Yoo
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
Quoted 7 lines
Quoted 3 lines
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]Oh sorry, I didn't understand why 0 was safer when I was reading it.
Quoted 7 lines
Quoted 2 lines
Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?
Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.
Hmm, but how paranoid do we have to be? Patch 1 converts SLAB to use kmalloc_large(). So it's now legitimate to have objects allocated by SLAB's kmalloc() that don't have a slab folio flag set, and their size is folio_size(). It would be more common than getting a bogus pointer, so should we return 0 just because a bogus pointer is possible? If we do that, then KASAN will fail to unpoison legitimate kmalloc_large() objects, no? What I suggested earlier is we could make the checks more precise - if folio_size() is smaller or equal order-1 page, then it's bogus because we only do kmalloc_large() for >order-1. If the object pointer is not to the beginning of the folio, then it's bogus, because kmalloc_large() returns the beginning of the folio. Then in these case we return 0, but otherwise we should return folio_size()?
Quoted 12 lines
The intuition here is that the caller has a pointer to an invalid object, and wants to use
ksize()to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by havingksize()return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj);touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)
Quoted 32 lines · Vlastimil Babka
Quoted 22 lines · Marco Elver
Quoted 21 lines · Hyeonggon Yoo
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
Quoted 7 lines
Quoted 3 lines
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]Oh sorry, I didn't understand why 0 was safer when I was reading it.
Quoted 7 lines
Quoted 2 lines
Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?
Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.
Quoted 5 lines
Hmm, but how paranoid do we have to be? Patch 1 converts SLAB to use
kmalloc_large(). So it's now legitimate to have objects allocated by SLAB'skmalloc()that don't have a slab folio flag set, and their size isfolio_size(). It would be more common than getting a bogus pointer, so should we return 0 just because a bogus pointer is possible?
No of course not, which is why I asked in the earlier email if it's a "definitive failure case".
Quoted 22 lines
If we do that,
then KASAN will fail to unpoison legitimatekmalloc_large()objects, no? What I suggested earlier is we could make the checks more precise - iffolio_size()is smaller or equal order-1 page, then it's bogus because we only dokmalloc_large()for >order-1. If the object pointer is not to the beginning of the folio, then it's bogus, becausekmalloc_large()returns the beginning of the folio. Then in these case we return 0, but otherwise we should returnfolio_size()?Quoted 12 lines
The intuition here is that the caller has a pointer to an invalid object, and wants to use
ksize()to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by havingksize()return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj);touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)
Quoted 30 lines · Marco Elver
Quoted 15 lines · Hyeonggon Yoo
Quoted 13 lines · Marco Elver
Quoted 11 lines · Hyeonggon Yoo
Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!
BTW, I have no idea what
__ksize()should return when an object that is not allocated from slab is passed. both 0 andfolio_size()seems wrong to me.Quoted 8 lines
Quoted 3 lines
Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]Oh sorry, I didn't understand why 0 was safer when I was reading it.
Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?
Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.
Quoted 8 lines
The intuition here is that the caller has a pointer to an invalid object, and wants to use
ksize()to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by havingksize()return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj);touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.
Oh, it's to prevent to corrupt memory further in failure case, like memset(obj, 0, s);
Quoted 3 lines
The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)
and making it able to check errors by caller.
Thank you so much for kind explanation.
I'll add what Vlastimil suggested in next series. Thanks!
Signature · 2 lines
Thank you, You are awesome! Hyeonggon :-)