mm/slab: save memory by allocating slabobj_ext array from leftover
Subsystems (2)
RFC PATCH 1/1 mm/slab: save memory by allocating slabobj_ext array from leftover
The leftover space in a slab is always smaller than s->size, and kmem caches for large objects that are not power-of-two sizes tend to have a greater amount of leftover space per slab. In some cases, the leftover space is larger than the size of the slabobj_ext array for the slab.
An excellent example of such a cache is ext4_inode_cache. On my system, the object size is 1144, with a preferred order of 3, 28 objects per slab, and 736 bytes of leftover space per slab.
Since the size of the slabobj_ext array is only 224 bytes (w/o mem profiling) or 448 bytes (w/ mem profiling) per slab, the entire array fits within the leftover space.
Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.
Enjoy the memory savings!
[ MEMCG=y, MEM_ALLOC_PROFILING=y ]
Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kB
After patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)
[ MEMCG=y, MEM_ALLOC_PROFILING=n ]
Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kB
After patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)
This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.
Enjoy the memory savings!
---
KASAN folks: Should we also poison the array before freeing the slab? If so, which API would be appropriate to use?
mm/slub.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-)
mm/slub.c +87 -8
@@ -785,6 +785,49 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)return *(unsigned int *)p; } +#ifdef CONFIG_SLAB_OBJ_EXT +static inline unsigned int obj_exts_size (struct slab *slab) +{ +return sizeof (struct slabobj_ext ) * slab->objects; +} + +static unsigned long obj_exts_offset (struct kmem_cache *s, +struct slab *slab) +{ +unsigned long objext_offset; + + objext_offset = s->red_left_pad + s->size * slab->objects; + objext_offset =ALIGN (objext_offset,sizeof (struct slabobj_ext )); +return objext_offset; +} + +static bool can_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +unsigned long objext_offset =obj_exts_offset (s, slab); +unsigned long objext_size =obj_exts_size (slab); + +return objext_offset + objext_size <=slab_size (slab); +} +#else +static inline unsigned int obj_exts_size (struct slab *slab) +{ +return 0 ; +} + +static unsigned long obj_exts_offset (struct kmem_cache *s, +struct slab *slab) +{ +return 0 ; +} + +static inline bool can_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +return false; +} +#endif +#ifdef CONFIG_SLUB_DEBUG static unsigned long object_map[BITS_TO_LONGS (MAX_OBJS_PER_PAGE )];static DEFINE_SPINLOCK (object_map_lock);
@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) start =slab_address (slab); length =slab_size (slab); end = start + length; - remainder = length % s->size; + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + remainder = length; + remainder -=obj_exts_offset (s, slab); + remainder -=obj_exts_size (slab); + }else { + remainder = length % s->size; + } +if (!remainder)return ;
@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab) slab->obj_exts =0 ; } +static void try_to_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +if (can_alloc_obj_exts_from_leftover (s, slab)) { +void *addr =slab_address (slab) +obj_exts_offset (s, slab); + + slab->obj_exts = (unsigned long )addr; +kasan_unpoison_range (addr,obj_exts_size (slab)); +memset (addr,0 ,obj_exts_size (slab)); +#ifdef CONFIG_MEMCG + slab->obj_exts |=MEMCG_DATA_OBJEXTS ; +#endif + } +} +static inline bool need_slab_obj_ext (void ) {if (mem_alloc_profiling_enabled ())
@@ -2077,6 +2143,11 @@ static inline void free_slab_obj_exts(struct slab *slab) { } +static inline void try_to_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +} +static inline bool need_slab_obj_ext (void ) {return false;
@@ -2592,7 +2663,9 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) static __always_inline void account_slab(struct slab *slab, int order,struct kmem_cache *s,gfp_t gfp) { -if (memcg_kmem_online () && (s->flags &SLAB_ACCOUNT )) +if (memcg_kmem_online () && + (s->flags &SLAB_ACCOUNT ) && + !slab_obj_exts (slab))alloc_slab_obj_exts (slab, s, gfp, true); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
@@ -2602,11 +2675,16 @@ static __always_inline void account_slab(struct slab *slab, int order, static __always_inline void unaccount_slab(struct slab *slab, int order,struct kmem_cache *s) { -if (memcg_kmem_online () ||need_slab_obj_ext ()) -free_slab_obj_exts (slab); -mod_node_page_state (slab_pgdat (slab),cache_vmstat_idx (s), -(PAGE_SIZE << order)); + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + slab->obj_exts =0 ; +return ; + } + +if (memcg_kmem_online () ||need_slab_obj_ext ()) +free_slab_obj_exts (slab); }static struct slab *allocate_slab (struct kmem_cache *s,gfp_t flags,int node)
@@ -2647,9 +2725,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->objects =oo_objects (oo); slab->inuse =0 ; slab->frozen =0 ; -init_slab_obj_exts (slab); - -account_slab (slab,oo_order (oo), s, flags); slab->slab_cache = s;
@@ -2658,6 +2733,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) start =slab_address (slab);setup_slab_debug (s, slab, start); +init_slab_obj_exts (slab); +/* Initialize the slabobj_ext array after poisoning the slab */ +try_to_alloc_obj_exts_from_leftover (s, slab); +account_slab (slab,oo_order (oo), s, flags); shuffle =shuffle_freelist (s, slab);
Signature · 1 line
2.43.0
Quoted 9 lines · Harry Yoo
The leftover space in a slab is always smaller than
s->size, and kmem caches for large objects that are not power-of-two sizes tend to have a greater amount of leftover space per slab. In some cases, the leftover space is larger than the size of theslabobj_extarray for the slab.An excellent example of such a cache is
ext4_inode_cache. On my system, the object size is 1144, with a preferred order of 3, 28 objects per slab, and 736 bytes of leftover space per slab.
Quoted 8 lines
Since the size of the
slabobj_extarray is only 224 bytes (w/o mem profiling) or 448 bytes (w/ mem profiling) per slab, the entire array fits within the leftover space.Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.Enjoy the memory savings!
Oops, I put this sentence twice in the changelog ;)
There's also a build error I missed when both MEMCG and MEM_ALLOC_PROFILING are not configured:
hyeyoo@hyeyoo ~/slab-misc (slab/for-next)> make -j24 mm/slub.o DESCEND objtool CALL scripts/checksyscalls.sh INSTALL libsubcmd_headers CC mm/slub.o mm/slub.c: In function ‘unaccount_slab’: mm/slub.c:2682:23: error: ‘struct slab’ has no member named ‘obj_exts’; did you mean ‘objects’? 2682 | slab->obj_exts = 0; | ^~~~~~~~ | objects make[3]: *** [scripts/Makefile.build:203: mm/slub.o] Error 1 make[2]: *** [scripts/Makefile.build:461: mm] Error 2 make[1]: *** [/home/hyeyoo/slab-misc/Makefile:2011: .] Error 2 make: *** [Makefile:248: __sub-make] Error 2
Will fix it in the next revision, but let me wait a bit for some feedback.
Quoted 197 lines
[ MEMCG=y, MEM_ALLOC_PROFILING=y ]
Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kBAfter patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)[ MEMCG=y, MEM_ALLOC_PROFILING=n ]
Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kBAfter patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.
Enjoy the memory savings!
Signed-off-by Harry Yoo <[email protected]>---
KASAN folks: Should we also poison the array before freeing the slab? If so, which API would be appropriate to use?mm/slub.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-)mm/slub.c +87
-8@@ -785,6 +785,49 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)return *(unsigned int *)p; } +#ifdef CONFIG_SLAB_OBJ_EXT +static inline unsigned int obj_exts_size (struct slab *slab) +{ +return sizeof (struct slabobj_ext ) * slab->objects; +} + +static unsigned long obj_exts_offset (struct kmem_cache *s, +struct slab *slab) +{ +unsigned long objext_offset; + + objext_offset = s->red_left_pad + s->size * slab->objects; + objext_offset =ALIGN (objext_offset,sizeof (struct slabobj_ext )); +return objext_offset; +} + +static bool can_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +unsigned long objext_offset =obj_exts_offset (s, slab); +unsigned long objext_size =obj_exts_size (slab); + +return objext_offset + objext_size <=slab_size (slab); +} +#else +static inline unsigned int obj_exts_size (struct slab *slab) +{ +return 0 ; +} + +static unsigned long obj_exts_offset (struct kmem_cache *s, +struct slab *slab) +{ +return 0 ; +} + +static inline bool can_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +return false; +} +#endif +#ifdef CONFIG_SLUB_DEBUG static unsigned long object_map[BITS_TO_LONGS (MAX_OBJS_PER_PAGE )];static DEFINE_SPINLOCK (object_map_lock);@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) start =slab_address (slab); length =slab_size (slab); end = start + length; - remainder = length % s->size; + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + remainder = length; + remainder -=obj_exts_offset (s, slab); + remainder -=obj_exts_size (slab); + }else { + remainder = length % s->size; + } +if (!remainder)return ;@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab) slab->obj_exts =0 ; } +static void try_to_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +if (can_alloc_obj_exts_from_leftover (s, slab)) { +void *addr =slab_address (slab) +obj_exts_offset (s, slab); + + slab->obj_exts = (unsigned long )addr; +kasan_unpoison_range (addr,obj_exts_size (slab)); +memset (addr,0 ,obj_exts_size (slab)); +#ifdef CONFIG_MEMCG + slab->obj_exts |=MEMCG_DATA_OBJEXTS ; +#endif + } +} +static inline bool need_slab_obj_ext (void ) {if (mem_alloc_profiling_enabled ())@@ -2077,6 +2143,11 @@ static inline void free_slab_obj_exts(struct slab *slab) { } +static inline void try_to_alloc_obj_exts_from_leftover (struct kmem_cache *s, +struct slab *slab) +{ +} +static inline bool need_slab_obj_ext (void ) {return false;@@ -2592,7 +2663,9 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) static __always_inline void account_slab(struct slab *slab, int order,struct kmem_cache *s,gfp_t gfp) { -if (memcg_kmem_online () && (s->flags &SLAB_ACCOUNT )) +if (memcg_kmem_online () && + (s->flags &SLAB_ACCOUNT ) && + !slab_obj_exts (slab))alloc_slab_obj_exts (slab, s, gfp, true); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),@@ -2602,11 +2675,16 @@ static __always_inline void account_slab(struct slab *slab, int order, static __always_inline void unaccount_slab(struct slab *slab, int order,struct kmem_cache *s) { -if (memcg_kmem_online () ||need_slab_obj_ext ()) -free_slab_obj_exts (slab); -mod_node_page_state (slab_pgdat (slab),cache_vmstat_idx (s), -(PAGE_SIZE << order)); + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + slab->obj_exts =0 ; +return ; + } + +if (memcg_kmem_online () ||need_slab_obj_ext ()) +free_slab_obj_exts (slab); }static struct slab *allocate_slab (struct kmem_cache *s,gfp_t flags,int node)@@ -2647,9 +2725,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->objects =oo_objects (oo); slab->inuse =0 ; slab->frozen =0 ; -init_slab_obj_exts (slab); - -account_slab (slab,oo_order (oo), s, flags); slab->slab_cache = s;@@ -2658,6 +2733,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) start =slab_address (slab);setup_slab_debug (s, slab, start); +init_slab_obj_exts (slab); +/* Initialize the slabobj_ext array after poisoning the slab */ +try_to_alloc_obj_exts_from_leftover (s, slab); +account_slab (slab,oo_order (oo), s, flags); shuffle =shuffle_freelist (s, slab);Signature · 1 line
2.43.0
Signature · 2 lines
Cheers, Harry / Hyeonggon
Hi Harry,
[...]
Quoted 29 lines
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.Enjoy the memory savings!
[ MEMCG=y, MEM_ALLOC_PROFILING=y ]
Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kBAfter patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)[ MEMCG=y, MEM_ALLOC_PROFILING=n ]
Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kBAfter patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)
Quoted 4 lines
This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.
Enjoy the memory savings!
Awesome :)
[...]
Quoted 21 lines
#ifdef
CONFIG_SLUB_DEBUG
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; staticDEFINE_SPINLOCK(object_map_lock);mm/slub.c +9
-1@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) start =slab_address (slab); length =slab_size (slab); end = start + length; - remainder = length % s->size; + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + remainder = length; + remainder -=obj_exts_offset (s, slab); + remainder -=obj_exts_size (slab); + }else { + remainder = length % s->size; + } +if (!remainder)return ;
mm/slub.c static noinline void free_slab_obj_exts(struct slab *slab) · line 2101
Quoted 2 lines
mm/slub.c +0
-0@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab) slab->obj_exts =0 ; }
What concerns me about this patch is the case where !memcg_kmem_online() and MEM_ALLOC_PROFILING is not used.
With this patch, obj_ext can still be created even in that situation, and as a result, if data is overwritten in the region previously padded with POISON_INUSE (before the patch), slab_pad_check() may no longer catch it
If this's ignorable, feel free toadd :
--
Sincerely,
Yeoreum Yun
IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.
Quoted 3 lines · Yeoreum Yun
Hi Harry,
[...]
Hi Yeoreum,
Quoted 29 lines
Quoted 29 lines
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.Enjoy the memory savings!
[ MEMCG=y, MEM_ALLOC_PROFILING=y ]
Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kBAfter patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)[ MEMCG=y, MEM_ALLOC_PROFILING=n ]
Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kBAfter patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)
Quoted 6 lines
Quoted 4 lines
This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.
Enjoy the memory savings!
Awesome :)
Thanks :)
Quoted 21 lines
[...]
Quoted 20 lines
#ifdef
CONFIG_SLUB_DEBUG
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; staticDEFINE_SPINLOCK(object_map_lock);mm/slub.c +9
-1@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab) start =slab_address (slab); length =slab_size (slab); end = start + length; - remainder = length % s->size; + +if (can_alloc_obj_exts_from_leftover (s, slab)) { + remainder = length; + remainder -=obj_exts_offset (s, slab); + remainder -=obj_exts_size (slab); + }else { + remainder = length % s->size; + } +if (!remainder)return ;
Quoted 9 lines
mm/slub.c static noinline void free_slab_obj_exts(struct slab *slab) · line 2101
Quoted 3 lines
mm/slub.c +0
-0@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab) slab->obj_exts =0 ; }What concerns me about this patch is the case where !memcg_kmem_online() and
MEM_ALLOC_PROFILINGis not used.
With this patch,obj_extcan still be created even in that situation, and as a result, if data is overwritten in the region previously padded withPOISON_INUSE(before the patch),slab_pad_check()may no longer catch it
That's a valid point.
I think allocating the array from the leftover space can be deferred until either MEMCG or MEM_ALLOC_PROFILING actually requests it.
Quoted 3 lines
If this's ignorable, feel free toadd :
Reviewed-by Yeoreum Yun <[email protected]>
That means the implementation will change a bit, so it's better to drop the R-b tag as the new change may invalidate "Looks good to me" state.
I'll Cc you in the next version—please take a look and review the updated version.
Thanks for reviewing!
Signature · 2 lines
Cheers, Harry / Hyeonggon
Christoph Lameter (Ampere)
reviewerQuoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.
How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.
There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.
Signature · 2 lines
Cheers, Harry / Hyeonggon
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Quoted 6 lines
Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.
There is no per-object metadata stored there and SLUB simply poisons the area. I taught
slab_pad_check()to skip checking theslabobj_extsarray.
Actually, it might be worth considering placing it within object in case the object is exactly a power-of-two size but has some wasted bytes due to alignment.
xfs_inode seems to be a good candidate that could benefit from this.
Hopefully it doesn't introduce too much complexity...
Signature · 2 lines
Cheers, Harry / Hyeonggon
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Quoted 6 lines
Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.
There is no per-object metadata stored there and SLUB simply poisons the area. I taught
slab_pad_check()to skip checking theslabobj_extsarray.
I can imagine going further with this in case where leftover space in slab isn't enough.
- indeed use per-object padding to store only single object's slabobj_ext, if it doesn't lead to memory waste
- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?
- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)
Of course having multiple variants would risk slower code, so fast paths should not be affected - we could have pointer to the 0th slabobj_ext (we already have) and now also stride (to support the "per-object padding case"
- there's still space in struct slab right?) and then the object alloc/free case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.
Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused, so would be only applicable with SLAB_ACCOUNT caches (if kmemcg is active) or when memalloc profiling is active. Shouldn't be a big issue as ad-hoc __GFP_ACCOUNT is handled by different cache selection for kmalloc() and I don't know if anyone is actually doing ad-hoc __GFP_ACCOUNT on named caches.
Quoted 10 lines · Vlastimil Babka
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Quoted 6 lines
Quoted 6 lines
Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.
There is no per-object metadata stored there and SLUB simply poisons the area. I taught
slab_pad_check()to skip checking theslabobj_extsarray.
Apologies for the late reply. I was sidetracked with multiple things :(
This is definitely worth optimizing, so let me make some progress even if it's a bit slow.
Quoted 2 lines
I can imagine going further with this in case where leftover space in slab isn't enough.
Right.
Quoted 1 line
- indeed use per-object padding to store only single object's
slabobj_ext,
I think the most conservative approach is to not increase object_size but use wasted area when ALIGN(size, align) is bigger than object_size.
A good candidate for that is xfs inode cache.
Quoted 1 line
if it doesn't lead to memory waste
You mean increasing object size but without decreasing the number of objects per slab?
...or (maybe) reducing the number of objects but without increasing the size of the remainder (same as calculated in calc_slab_order())?
Quoted 2 lines
- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?
If object size is small the array likely does not fit in one object...
Quoted 2 lines
- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)
It can be tried, but variable struct slab size may or may not work. That'll depend on the implementation details of how we allocate struct slab in the future.
Quoted 2 lines
Of course having multiple variants would risk slower code, so fast paths should not be affected
I agree that affecting fastpath is not great.
Quoted 4 lines
we could have pointer to the 0th
slabobj_ext(we
already have) and now also stride (to support the "per-object padding case"- there's still space in struct slab right?)
which space are you referring to, maybe lower 16 bits of page_type?
Quoted 4 lines
and then the object alloc/free
case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.
So it would be something like (please correct if I misunderstood):
index = obj_to_index(s, slab, object)
(the struct slabobj_ext pointer for the object at given index)
== slab->obj_exts + stride * index
slab->obj_exts, stride are determined depending on the case:
- In the normal case (the array is allocated from kmalloc caches), stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the address of the buffer allocated from kmalloc)
- In "the obj_exts array is stored in the leftover space" case, stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the start address of the leftover space)
- In "per-object padding" case,
stride = s->size
slab->obj_exts = slab_address(slab) + s->red_left_pad +
(offset of slabobj_ext);
Ok, I think it will work. Great idea!
Quoted 2 lines
Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused
Right.
Quoted 3 lines
so would be only applicable
withSLAB_ACCOUNTcaches (if kmemcg is active) or when memalloc profiling is active.
Right.
Quoted 3 lines
Shouldn't be a big issue as ad-hoc
__GFP_ACCOUNTis handled by different cache selection forkmalloc()and I don't know if anyone is actually doing ad-hoc__GFP_ACCOUNTon named caches.
A while ago I was thinking of getting rid of ad-hoc __GFP_ACCOUNT usage for slab allocations, but at least xarray cache appears to use it in an ad-hoc manner. (See xas_nomem()).
In that case, not all allocations to the same cache has __GFP_ACCOUNT flag set.
Signature · 2 lines
Cheers, Harry / Hyeonggon
Quoted 26 lines · Harry Yoo
Quoted 10 lines · Vlastimil Babka
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo
Allocate
slabobj_extsarray from this unused space instead of usingkcalloc(), when it is large enough.How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.
Quoted 6 lines
Quoted 6 lines
Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.
There is no per-object metadata stored there and SLUB simply poisons the area. I taught
slab_pad_check()to skip checking theslabobj_extsarray.Apologies for the late reply. I was sidetracked with multiple things :(
This is definitely worth optimizing, so let me make some progress even if it's a bit slow.
Quoted 2 lines
I can imagine going further with this in case where leftover space in slab isn't enough.
Quoted 8 lines
Right.
Quoted 1 line
- indeed use per-object padding to store only single object's
slabobj_ext,I think the most conservative approach is to not increase
object_sizebut use wasted area whenALIGN(size, align)is bigger thanobject_size.A good candidate for that is xfs inode cache.
Yeah that's what I meant.
Quoted 4 lines
Quoted 1 line
if it doesn't lead to memory waste
You mean increasing object size but without decreasing the number of objects per slab?
Just the ALIGN thing. It we can increase size without decreasing objects it means we can also just use leftover space for the array, IIUC?
Quoted 7 lines
...or (maybe) reducing the number of objects but without increasing the size of the remainder (same as calculated in
calc_slab_order())?Quoted 2 lines
- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?
If object size is small the array likely does not fit in one object...
Hmm perhaps.
Quoted 11 lines
Quoted 2 lines
- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)
It can be tried, but variable struct slab size may or may not work. That'll depend on the implementation details of how we allocate struct slab in the future.
Quoted 2 lines
Of course having multiple variants would risk slower code, so fast paths should not be affected
I agree that affecting fastpath is not great.
So we'll have to measure it.
Quoted 6 lines
Quoted 4 lines
we could have pointer to the 0th
slabobj_ext(we
already have) and now also stride (to support the "per-object padding case"- there's still space in struct slab right?)
which space are you referring to, maybe lower 16 bits of
page_type?
Ugh I thought counters used only 32bit of 64... so the space exists only for 64bit kernels? It would be fine to limit the optimization to those only.
Quoted 10 lines
Quoted 4 lines
and then the object alloc/free
case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.So it would be something like (please correct if I misunderstood):
index =
obj_to_index(s, slab, object)
(the structslabobj_extpointer for the object at given index)
==slab->obj_exts+ stride * index
Yeah.
Quoted 11 lines
slab->obj_exts, stride are determined depending on the case:- In the normal case (the array is allocated from kmalloc caches), stride =
sizeof(struct slabobj_ext)
slab->obj_exts= (the address of the buffer allocated from kmalloc)- In "the
obj_extsarray is stored in the leftover space" case, stride =sizeof(struct slabobj_ext)
slab->obj_exts= (the start address of the leftover space)
Quoted 6 lines
- In "per-object padding" case,
stride =s->size
slab->obj_exts=slab_address(slab)+s->red_left_pad+(offset of slabobj_ext);Ok, I think it will work. Great idea!
Yeah.
Quoted 16 lines
Quoted 3 lines
Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused
Right.
Quoted 3 lines
so would be only applicable
withSLAB_ACCOUNTcaches (if kmemcg is active) or when memalloc profiling is active.Right.
Quoted 3 lines
Shouldn't be a big issue as ad-hoc
__GFP_ACCOUNTis handled by different cache selection forkmalloc()and I don't know if anyone is actually doing ad-hoc__GFP_ACCOUNTon named caches.
Quoted 6 lines
A while ago I was thinking of getting rid of ad-hoc
__GFP_ACCOUNTusage for slab allocations, but at least xarray cache appears to use it in an ad-hoc manner. (Seexas_nomem()).In that case, not all allocations to the same cache has
__GFP_ACCOUNTflag set.
Hmm bummer.
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Harry Yoo
Quoted 5 lines · Vlastimil Babka
Quoted 2 lines · Harry Yoo
Quoted 1 line · Christoph Lameter (Ampere)
On Fri, 13 Jun 2025, Harry Yoo wrote:
we could have pointer to the 0th
slabobj_ext(we
already have) and now also stride (to support the "per-object padding case"
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
- there's still space in struct slab right?)
which space are you referring to, maybe lower 16 bits of
page_type?Ugh I thought counters used only 32bit of 64... so the space exists only for 64bit kernels? It would be fine to limit the optimization to those only.
Oh, you're right and it seems to be better than using page_type. RFC v2 uses page_type but I'll adjust this in V3.
Signature · 2 lines
Cheers, Harry / Hyeonggon