Skip to contentSkip to conversation

mm/slab: save memory by allocating slabobj_ext array from leftover

RFC PATCH In review Harry Yoo 4 participants 11 messages Jun 13, 2025 – Aug 27, 2025

Subsystems (2)
lore
Share & tools
lore

Harry Yoo

author
#0 lore raw

RFC PATCH 1/1 mm/slab: save memory by allocating slabobj_ext array from leftover

The leftover space in a slab is always smaller than s->size, and kmem caches for large objects that are not power-of-two sizes tend to have a greater amount of leftover space per slab. In some cases, the leftover space is larger than the size of the slabobj_ext array for the slab.

An excellent example of such a cache is ext4_inode_cache. On my system, the object size is 1144, with a preferred order of 3, 28 objects per slab, and 736 bytes of leftover space per slab.

Since the size of the slabobj_ext array is only 224 bytes (w/o mem profiling) or 448 bytes (w/ mem profiling) per slab, the entire array fits within the leftover space.

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

Enjoy the memory savings!

[ MEMCG=y, MEM_ALLOC_PROFILING=y ]

Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kB

After patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)

[ MEMCG=y, MEM_ALLOC_PROFILING=n ]

Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kB

After patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)

This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.

Enjoy the memory savings!

---
KASAN folks: Should we also poison the array before freeing the slab? If so, which API would be appropriate to use?

 mm/slub.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 87 insertions(+), 8 deletions(-)
mm/slub.c +87 -8
index cf3637324243..20f0f76f0c65 100644
@@ -785,6 +785,49 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
 	return *(unsigned int *)p;
 }
 
+#ifdef CONFIG_SLAB_OBJ_EXT
+static inline unsigned int obj_exts_size(struct slab *slab)
+{
+	return sizeof(struct slabobj_ext) * slab->objects;
+}
+
+static unsigned long obj_exts_offset(struct kmem_cache *s,
+				    struct slab *slab)
+{
+	unsigned long objext_offset;
+
+	objext_offset = s->red_left_pad + s->size * slab->objects;
+	objext_offset = ALIGN(objext_offset, sizeof(struct slabobj_ext));
+	return objext_offset;
+}
+
+static bool can_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+					     struct slab *slab)
+{
+	unsigned long objext_offset = obj_exts_offset(s, slab);
+	unsigned long objext_size = obj_exts_size(slab);
+
+	return objext_offset + objext_size <= slab_size(slab);
+}
+#else
+static inline unsigned int obj_exts_size(struct slab *slab)
+{
+	return 0;
+}
+
+static unsigned long obj_exts_offset(struct kmem_cache *s,
+				    struct slab *slab)
+{
+	return 0;
+}
+
+static inline bool can_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						    struct slab *slab)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_SLUB_DEBUG
 static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
 static DEFINE_SPINLOCK(object_map_lock);
@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
 	start = slab_address(slab);
 	length = slab_size(slab);
 	end = start + length;
-	remainder = length % s->size;
+
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		remainder = length;
+		remainder -= obj_exts_offset(s, slab);
+		remainder -= obj_exts_size(slab);
+	} else {
+		remainder = length % s->size;
+	}
+
 	if (!remainder)
 		return;
 
@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab)
 	slab->obj_exts = 0;
 }
 
+static void try_to_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						struct slab *slab)
+{
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		void *addr = slab_address(slab) + obj_exts_offset(s, slab);
+
+		slab->obj_exts = (unsigned long)addr;
+		kasan_unpoison_range(addr, obj_exts_size(slab));
+		memset(addr, 0, obj_exts_size(slab));
+#ifdef CONFIG_MEMCG
+		slab->obj_exts |= MEMCG_DATA_OBJEXTS;
+#endif
+	}
+}
+
 static inline bool need_slab_obj_ext(void)
 {
 	if (mem_alloc_profiling_enabled())
@@ -2077,6 +2143,11 @@ static inline void free_slab_obj_exts(struct slab *slab)
 {
 }
 
+static inline void try_to_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						       struct slab *slab)
+{
+}
+
 static inline bool need_slab_obj_ext(void)
 {
 	return false;
@@ -2592,7 +2663,9 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
 static __always_inline void account_slab(struct slab *slab, int order,
 					 struct kmem_cache *s, gfp_t gfp)
 {
-	if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
+	if (memcg_kmem_online() &&
+			(s->flags & SLAB_ACCOUNT) &&
+			!slab_obj_exts(slab))
 		alloc_slab_obj_exts(slab, s, gfp, true);
 
 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
@@ -2602,11 +2675,16 @@ static __always_inline void account_slab(struct slab *slab, int order,
 static __always_inline void unaccount_slab(struct slab *slab, int order,
 					   struct kmem_cache *s)
 {
-	if (memcg_kmem_online() || need_slab_obj_ext())
-		free_slab_obj_exts(slab);
-
 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
 			    -(PAGE_SIZE << order));
+
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		slab->obj_exts = 0;
+		return;
+	}
+
+	if (memcg_kmem_online() || need_slab_obj_ext())
+		free_slab_obj_exts(slab);
 }
 
 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
@@ -2647,9 +2725,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 	slab->objects = oo_objects(oo);
 	slab->inuse = 0;
 	slab->frozen = 0;
-	init_slab_obj_exts(slab);
-
-	account_slab(slab, oo_order(oo), s, flags);
 
 	slab->slab_cache = s;
 
@@ -2658,6 +2733,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 	start = slab_address(slab);
 
 	setup_slab_debug(s, slab, start);
+	init_slab_obj_exts(slab);
+	/* Initialize the slabobj_ext array after poisoning the slab */
+	try_to_alloc_obj_exts_from_leftover(s, slab);
+	account_slab(slab, oo_order(oo), s, flags);
 
 	shuffle = shuffle_freelist(s, slab);
 
Signature · 1 line
2.43.0
author
#1 lore raw
Quoted 9 lines · Harry Yoo

The leftover space in a slab is always smaller than s->size, and kmem caches for large objects that are not power-of-two sizes tend to have a greater amount of leftover space per slab. In some cases, the leftover space is larger than the size of the slabobj_ext array for the slab.

An excellent example of such a cache is ext4_inode_cache. On my system, the object size is 1144, with a preferred order of 3, 28 objects per slab, and 736 bytes of leftover space per slab.

Quoted 8 lines

Since the size of the slabobj_ext array is only 224 bytes (w/o mem profiling) or 448 bytes (w/ mem profiling) per slab, the entire array fits within the leftover space.

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

Enjoy the memory savings!

Oops, I put this sentence twice in the changelog ;)

There's also a build error I missed when both MEMCG and MEM_ALLOC_PROFILING are not configured:

	hyeyoo@hyeyoo ~/slab-misc (slab/for-next)> make -j24 mm/slub.o
	  DESCEND objtool
	  CALL    scripts/checksyscalls.sh
	  INSTALL libsubcmd_headers
	  CC      mm/slub.o
	mm/slub.c: In function ‘unaccount_slab’:
	mm/slub.c:2682:23: error: ‘struct slab’ has no member named ‘obj_exts’; did you mean ‘objects’?
	 2682 |                 slab->obj_exts = 0;
	      |                       ^~~~~~~~
	      |                       objects
	make[3]: *** [scripts/Makefile.build:203: mm/slub.o] Error 1
	make[2]: *** [scripts/Makefile.build:461: mm] Error 2
	make[1]: *** [/home/hyeyoo/slab-misc/Makefile:2011: .] Error 2
	make: *** [Makefile:248: __sub-make] Error 2

Will fix it in the next revision, but let me wait a bit for some feedback.

Quoted 197 lines

[ MEMCG=y, MEM_ALLOC_PROFILING=y ]

Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kB

After patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)

[ MEMCG=y, MEM_ALLOC_PROFILING=n ]

Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kB

After patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)

This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.

Enjoy the memory savings!

---
KASAN folks: Should we also poison the array before freeing the slab? If so, which API would be appropriate to use?

 mm/slub.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 87 insertions(+), 8 deletions(-)
mm/slub.c +87 -8
index cf3637324243..20f0f76f0c65 100644
@@ -785,6 +785,49 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
 	return *(unsigned int *)p;
 }
 
+#ifdef CONFIG_SLAB_OBJ_EXT
+static inline unsigned int obj_exts_size(struct slab *slab)
+{
+	return sizeof(struct slabobj_ext) * slab->objects;
+}
+
+static unsigned long obj_exts_offset(struct kmem_cache *s,
+				    struct slab *slab)
+{
+	unsigned long objext_offset;
+
+	objext_offset = s->red_left_pad + s->size * slab->objects;
+	objext_offset = ALIGN(objext_offset, sizeof(struct slabobj_ext));
+	return objext_offset;
+}
+
+static bool can_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+					     struct slab *slab)
+{
+	unsigned long objext_offset = obj_exts_offset(s, slab);
+	unsigned long objext_size = obj_exts_size(slab);
+
+	return objext_offset + objext_size <= slab_size(slab);
+}
+#else
+static inline unsigned int obj_exts_size(struct slab *slab)
+{
+	return 0;
+}
+
+static unsigned long obj_exts_offset(struct kmem_cache *s,
+				    struct slab *slab)
+{
+	return 0;
+}
+
+static inline bool can_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						    struct slab *slab)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_SLUB_DEBUG
 static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
 static DEFINE_SPINLOCK(object_map_lock);
@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
 	start = slab_address(slab);
 	length = slab_size(slab);
 	end = start + length;
-	remainder = length % s->size;
+
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		remainder = length;
+		remainder -= obj_exts_offset(s, slab);
+		remainder -= obj_exts_size(slab);
+	} else {
+		remainder = length % s->size;
+	}
+
 	if (!remainder)
 		return;
 
@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab)
 	slab->obj_exts = 0;
 }
 
+static void try_to_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						struct slab *slab)
+{
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		void *addr = slab_address(slab) + obj_exts_offset(s, slab);
+
+		slab->obj_exts = (unsigned long)addr;
+		kasan_unpoison_range(addr, obj_exts_size(slab));
+		memset(addr, 0, obj_exts_size(slab));
+#ifdef CONFIG_MEMCG
+		slab->obj_exts |= MEMCG_DATA_OBJEXTS;
+#endif
+	}
+}
+
 static inline bool need_slab_obj_ext(void)
 {
 	if (mem_alloc_profiling_enabled())
@@ -2077,6 +2143,11 @@ static inline void free_slab_obj_exts(struct slab *slab)
 {
 }
 
+static inline void try_to_alloc_obj_exts_from_leftover(struct kmem_cache *s,
+						       struct slab *slab)
+{
+}
+
 static inline bool need_slab_obj_ext(void)
 {
 	return false;
@@ -2592,7 +2663,9 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
 static __always_inline void account_slab(struct slab *slab, int order,
 					 struct kmem_cache *s, gfp_t gfp)
 {
-	if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
+	if (memcg_kmem_online() &&
+			(s->flags & SLAB_ACCOUNT) &&
+			!slab_obj_exts(slab))
 		alloc_slab_obj_exts(slab, s, gfp, true);
 
 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
@@ -2602,11 +2675,16 @@ static __always_inline void account_slab(struct slab *slab, int order,
 static __always_inline void unaccount_slab(struct slab *slab, int order,
 					   struct kmem_cache *s)
 {
-	if (memcg_kmem_online() || need_slab_obj_ext())
-		free_slab_obj_exts(slab);
-
 	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
 			    -(PAGE_SIZE << order));
+
+	if (can_alloc_obj_exts_from_leftover(s, slab)) {
+		slab->obj_exts = 0;
+		return;
+	}
+
+	if (memcg_kmem_online() || need_slab_obj_ext())
+		free_slab_obj_exts(slab);
 }
 
 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
@@ -2647,9 +2725,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 	slab->objects = oo_objects(oo);
 	slab->inuse = 0;
 	slab->frozen = 0;
-	init_slab_obj_exts(slab);
-
-	account_slab(slab, oo_order(oo), s, flags);
 
 	slab->slab_cache = s;
 
@@ -2658,6 +2733,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 	start = slab_address(slab);
 
 	setup_slab_debug(s, slab, start);
+	init_slab_obj_exts(slab);
+	/* Initialize the slabobj_ext array after poisoning the slab */
+	try_to_alloc_obj_exts_from_leftover(s, slab);
+	account_slab(slab, oo_order(oo), s, flags);
 
 	shuffle = shuffle_freelist(s, slab);
 
Signature · 1 line
2.43.0
Signature · 2 lines
Cheers,
Harry / Hyeonggon
Reviewed-by on v1 1/1
#2 lore raw
In reply to Harry Yoo

Hi Harry,

[...]

Quoted 29 lines

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

Enjoy the memory savings!

[ MEMCG=y, MEM_ALLOC_PROFILING=y ]

Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kB

After patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)

[ MEMCG=y, MEM_ALLOC_PROFILING=n ]

Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kB

After patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)

Quoted 4 lines

This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.

Enjoy the memory savings!

Awesome :)

[...]

Quoted 21 lines

#ifdef CONFIG_SLUB_DEBUG
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; static DEFINE_SPINLOCK(object_map_lock);

mm/slub.c +9 -1
@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
      start = slab_address(slab);
      length = slab_size(slab);
      end = start + length;
-     remainder = length % s->size;
+
+     if (can_alloc_obj_exts_from_leftover(s, slab)) {
+             remainder = length;
+             remainder -= obj_exts_offset(s, slab);
+             remainder -= obj_exts_size(slab);
+     } else {
+             remainder = length % s->size;
+     }
+
      if (!remainder)
              return;
 

mm/slub.c static noinline void free_slab_obj_exts(struct slab *slab) · line 2101

Quoted 2 lines
mm/slub.c +0 -0
@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab)
      slab->obj_exts = 0;
 }

What concerns me about this patch is the case where !memcg_kmem_online() and MEM_ALLOC_PROFILING is not used.
With this patch, obj_ext can still be created even in that situation, and as a result, if data is overwritten in the region previously padded with POISON_INUSE (before the patch), slab_pad_check() may no longer catch it

If this's ignorable, feel free toadd :

--
Sincerely,
Yeoreum Yun
IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.

author
#3 lore raw
Quoted 3 lines · Yeoreum Yun

Hi Harry,

[...]

Hi Yeoreum,

Quoted 29 lines
Quoted 29 lines

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

Enjoy the memory savings!

[ MEMCG=y, MEM_ALLOC_PROFILING=y ]

Before patch (run updatedb):
Slab: 5815196 kB
SReclaimable: 5042824 kB
SUnreclaim: 772372 kB

After patch (run updatedb):
Slab: 5748664 kB
SReclaimable: 5041608 kB
SUnreclaim: 707084 kB (-63.75 MiB)

[ MEMCG=y, MEM_ALLOC_PROFILING=n ]

Before patch (run updatedb):
Slab: 5637764 kB
SReclaimable: 5042428 kB
SUnreclaim: 595284 kB

After patch (run updatedb):
Slab: 5598992 kB
SReclaimable: 5042248 kB
SUnreclaim: 560396 kB (-34.07 MiB)

Quoted 6 lines
Quoted 4 lines

This saves from hundreds of KiBs up to several tens of MiBs of memory on my machine, depending on the config and slab memory usage.

Enjoy the memory savings!

Awesome :)

Thanks :)

Quoted 21 lines

[...]

Quoted 20 lines

#ifdef CONFIG_SLUB_DEBUG
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; static DEFINE_SPINLOCK(object_map_lock);

mm/slub.c +9 -1
@@ -1307,7 +1350,15 @@ slab_pad_check(struct kmem_cache *s, struct slab *slab)
      start = slab_address(slab);
      length = slab_size(slab);
      end = start + length;
-     remainder = length % s->size;
+
+     if (can_alloc_obj_exts_from_leftover(s, slab)) {
+             remainder = length;
+             remainder -= obj_exts_offset(s, slab);
+             remainder -= obj_exts_size(slab);
+     } else {
+             remainder = length % s->size;
+     }
+
      if (!remainder)
              return;
 
Quoted 9 lines

mm/slub.c static noinline void free_slab_obj_exts(struct slab *slab) · line 2101

Quoted 3 lines
mm/slub.c +0 -0
@@ -2049,6 +2100,21 @@ static noinline void free_slab_obj_exts(struct slab *slab)
      slab->obj_exts = 0;
 }

What concerns me about this patch is the case where !memcg_kmem_online() and MEM_ALLOC_PROFILING is not used.
With this patch, obj_ext can still be created even in that situation, and as a result, if data is overwritten in the region previously padded with POISON_INUSE (before the patch), slab_pad_check() may no longer catch it

That's a valid point.

I think allocating the array from the leftover space can be deferred until either MEMCG or MEM_ALLOC_PROFILING actually requests it.

Quoted 3 lines

If this's ignorable, feel free toadd :

That means the implementation will change a bit, so it's better to drop the R-b tag as the new change may invalidate "Looks good to me" state.

I'll Cc you in the next version—please take a look and review the updated version.

Thanks for reviewing!

Signature · 2 lines
Cheers,
Harry / Hyeonggon
#4 lore raw
In reply to Harry Yoo
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

author
#5 lore raw
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.

There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.

Signature · 2 lines
Cheers,
Harry / Hyeonggon
author
#6 lore raw
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

Quoted 6 lines

Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.

There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.

Actually, it might be worth considering placing it within object in case the object is exactly a power-of-two size but has some wasted bytes due to alignment.

xfs_inode seems to be a good candidate that could benefit from this.

Hopefully it doesn't introduce too much complexity...

Signature · 2 lines
Cheers,
Harry / Hyeonggon
#7 lore raw
In reply to Harry Yoo
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

Quoted 6 lines

Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.

There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.

I can imagine going further with this in case where leftover space in slab isn't enough.

- indeed use per-object padding to store only single object's slabobj_ext, if it doesn't lead to memory waste
- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?
- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)

Of course having multiple variants would risk slower code, so fast paths should not be affected - we could have pointer to the 0th slabobj_ext (we already have) and now also stride (to support the "per-object padding case"
- there's still space in struct slab right?) and then the object alloc/free case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.

Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused, so would be only applicable with SLAB_ACCOUNT caches (if kmemcg is active) or when memalloc profiling is active. Shouldn't be a big issue as ad-hoc __GFP_ACCOUNT is handled by different cache selection for kmalloc() and I don't know if anyone is actually doing ad-hoc __GFP_ACCOUNT on named caches.

author
#8 lore raw
Quoted 10 lines · Vlastimil Babka
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

Quoted 6 lines
Quoted 6 lines

Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.

There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.

Apologies for the late reply. I was sidetracked with multiple things :(

This is definitely worth optimizing, so let me make some progress even if it's a bit slow.

Quoted 2 lines

I can imagine going further with this in case where leftover space in slab isn't enough.

Right.

Quoted 1 line

- indeed use per-object padding to store only single object's slabobj_ext,

I think the most conservative approach is to not increase object_size but use wasted area when ALIGN(size, align) is bigger than object_size.

A good candidate for that is xfs inode cache.

Quoted 1 line

if it doesn't lead to memory waste

You mean increasing object size but without decreasing the number of objects per slab?

...or (maybe) reducing the number of objects but without increasing the size of the remainder (same as calculated in calc_slab_order())?

Quoted 2 lines

- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?

If object size is small the array likely does not fit in one object...

Quoted 2 lines

- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)

It can be tried, but variable struct slab size may or may not work. That'll depend on the implementation details of how we allocate struct slab in the future.

Quoted 2 lines

Of course having multiple variants would risk slower code, so fast paths should not be affected

I agree that affecting fastpath is not great.

Quoted 4 lines

we could have pointer to the 0th slabobj_ext (we
already have) and now also stride (to support the "per-object padding case"

- there's still space in struct slab right?)

which space are you referring to, maybe lower 16 bits of page_type?

Quoted 4 lines

and then the object alloc/free
case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.

So it would be something like (please correct if I misunderstood):

index = obj_to_index(s, slab, object)
(the struct slabobj_ext pointer for the object at given index)
== slab->obj_exts + stride * index

slab->obj_exts, stride are determined depending on the case:

- In the normal case (the array is allocated from kmalloc caches), stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the address of the buffer allocated from kmalloc)

- In "the obj_exts array is stored in the leftover space" case, stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the start address of the leftover space)

- In "per-object padding" case,
stride = s->size
slab->obj_exts = slab_address(slab) + s->red_left_pad +

                       (offset of slabobj_ext);

Ok, I think it will work. Great idea!

Quoted 2 lines

Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused

Right.

Quoted 3 lines

so would be only applicable
with SLAB_ACCOUNT caches (if kmemcg is active) or when memalloc profiling is active.

Right.

Quoted 3 lines

Shouldn't be a big issue as ad-hoc __GFP_ACCOUNT is handled by different cache selection for kmalloc() and I don't know if anyone is actually doing ad-hoc __GFP_ACCOUNT on named caches.

A while ago I was thinking of getting rid of ad-hoc __GFP_ACCOUNT usage for slab allocations, but at least xarray cache appears to use it in an ad-hoc manner. (See xas_nomem()).

In that case, not all allocations to the same cache has __GFP_ACCOUNT flag set.

Signature · 2 lines
Cheers,
Harry / Hyeonggon
#9 lore raw
Quoted 26 lines · Harry Yoo
Quoted 10 lines · Vlastimil Babka
Quoted 9 lines · Harry Yoo
Quoted 7 lines · Christoph Lameter (Ampere)
Quoted 2 lines · Harry Yoo

Allocate slabobj_exts array from this unused space instead of using kcalloc(), when it is large enough.

How does slab debug work in this case? The object layout gets a bit complicated with other metadata there as well.

Quoted 6 lines
Quoted 6 lines

Oh, the 'leftover' space I mentioned the cover letter refers to the wasted space after the last object in a slab, not unused bytes within objects.

There is no per-object metadata stored there and SLUB simply poisons the area. I taught slab_pad_check() to skip checking the slabobj_exts array.

Apologies for the late reply. I was sidetracked with multiple things :(

This is definitely worth optimizing, so let me make some progress even if it's a bit slow.

Quoted 2 lines

I can imagine going further with this in case where leftover space in slab isn't enough.

Quoted 8 lines

Right.

Quoted 1 line

- indeed use per-object padding to store only single object's slabobj_ext,

I think the most conservative approach is to not increase object_size but use wasted area when ALIGN(size, align) is bigger than object_size.

A good candidate for that is xfs inode cache.

Yeah that's what I meant.

Quoted 4 lines
Quoted 1 line

if it doesn't lead to memory waste

You mean increasing object size but without decreasing the number of objects per slab?

Just the ALIGN thing. It we can increase size without decreasing objects it means we can also just use leftover space for the array, IIUC?

Quoted 7 lines

...or (maybe) reducing the number of objects but without increasing the size of the remainder (same as calculated in calc_slab_order())?

Quoted 2 lines

- if not possible, but object size is small enough so there are many per slab, maybe have one less object per slab to store the array?

If object size is small the array likely does not fit in one object...

Hmm perhaps.

Quoted 11 lines
Quoted 2 lines

- once we have struct slab decoupled from struct page, it could be part of struct slab directly (but it would mean struct slab isn't fixed size)

It can be tried, but variable struct slab size may or may not work. That'll depend on the implementation details of how we allocate struct slab in the future.

Quoted 2 lines

Of course having multiple variants would risk slower code, so fast paths should not be affected

I agree that affecting fastpath is not great.

So we'll have to measure it.

Quoted 6 lines
Quoted 4 lines

we could have pointer to the 0th slabobj_ext (we
already have) and now also stride (to support the "per-object padding case"

- there's still space in struct slab right?)

which space are you referring to, maybe lower 16 bits of page_type?

Ugh I thought counters used only 32bit of 64... so the space exists only for 64bit kernels? It would be fine to limit the optimization to those only.

Quoted 10 lines
Quoted 4 lines

and then the object alloc/free
case could be oblivious to the storage method, with just a bit more arithmetic (stride). Slab folio alloc/free would be more complicated but are not fath path.

So it would be something like (please correct if I misunderstood):

index = obj_to_index(s, slab, object)
(the struct slabobj_ext pointer for the object at given index)
== slab->obj_exts + stride * index

Yeah.

Quoted 11 lines

slab->obj_exts, stride are determined depending on the case:

- In the normal case (the array is allocated from kmalloc caches), stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the address of the buffer allocated from kmalloc)

- In "the obj_exts array is stored in the leftover space" case, stride = sizeof(struct slabobj_ext)
slab->obj_exts = (the start address of the leftover space)

Quoted 6 lines

- In "per-object padding" case,
stride = s->size
slab->obj_exts = slab_address(slab) + s->red_left_pad +

                       (offset of slabobj_ext);

Ok, I think it will work. Great idea!

Yeah.

Quoted 16 lines
Quoted 3 lines

Also some variants would be wasteful if they need to be decided upfront (the 2nd and 3rd above) and then the array is unused

Right.

Quoted 3 lines

so would be only applicable
with SLAB_ACCOUNT caches (if kmemcg is active) or when memalloc profiling is active.

Right.

Quoted 3 lines

Shouldn't be a big issue as ad-hoc __GFP_ACCOUNT is handled by different cache selection for kmalloc() and I don't know if anyone is actually doing ad-hoc __GFP_ACCOUNT on named caches.

Quoted 6 lines

A while ago I was thinking of getting rid of ad-hoc __GFP_ACCOUNT usage for slab allocations, but at least xarray cache appears to use it in an ad-hoc manner. (See xas_nomem()).

In that case, not all allocations to the same cache has __GFP_ACCOUNT flag set.

Hmm bummer.

author
#10 lore raw
Quoted 8 lines · Vlastimil Babka
Quoted 7 lines · Harry Yoo
Quoted 5 lines · Vlastimil Babka
Quoted 2 lines · Harry Yoo
Quoted 1 line · Christoph Lameter (Ampere)

On Fri, 13 Jun 2025, Harry Yoo wrote:

we could have pointer to the 0th slabobj_ext (we
already have) and now also stride (to support the "per-object padding case"

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

- there's still space in struct slab right?)

which space are you referring to, maybe lower 16 bits of page_type?

Ugh I thought counters used only 32bit of 64... so the space exists only for 64bit kernels? It would be fine to limit the optimization to those only.

Oh, you're right and it seems to be better than using page_type. RFC v2 uses page_type but I'll adjust this in V3.

Signature · 2 lines
Cheers,
Harry / Hyeonggon