Skip to contentSkip to conversation

slab cleanups

PATCH Superseded Hyeonggon Yoo 3 participants 21 messages Mar 4, 2022 – Mar 5, 2022

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover slab cleanups

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Hello, these are cleanup patches for slab.
Please consider them for slab-next :)

Any comments will be appreciated.
Thanks.

Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify __ksize()
mm/sl[auo]b: move definition of __ksize() to mm/slab.h mm/slub: limit number of node partial slabs only in cache creation mm/slub: refactor deactivate_slab()

 include/linux/slab.h |  36 ++++++------
 mm/slab.c            |  51 ++++++++---------
 mm/slab.h            |  21 +++++++
 mm/slab_common.c     |  20 +++++++
 mm/slob.c            |   1 -
 mm/slub.c            | 130 ++++++++++++-------------------------------
 6 files changed, 114 insertions(+), 145 deletions(-)
Signature · 1 line
2.33.1
#1 lore raw

PATCH v2 1/5 mm/slab: kmalloc: pass requests larger than order-1 page to page allocator

There is not much benefit for serving large objects in kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.

---
 include/linux/slab.h | 35 ++++++++++++++++-------------------
 mm/slab.c            | 31 +++++++++++++++++++++++++++----
 mm/slab.h            | 19 +++++++++++++++++++
 mm/slub.c            | 19 -------------------
 4 files changed, 62 insertions(+), 42 deletions(-)
include/linux/slab.h +16 -19
index 37bde99b74af..e7b3330db4f3 100644
@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);
  * Kmalloc array related definitions
  */
 
-#ifdef CONFIG_SLAB
 /*
- * The largest kmalloc size supported by the SLAB allocators is
- * 32 megabyte (2^25) or the maximum allocatable page order if that is
- * less than 32 MB.
- *
- * WARNING: Its not easy to increase this value since the allocators have
- * to do various tricks to work around compiler limitations in order to
- * ensure proper constant folding.
+ * SLAB and SLUB directly allocates requests fitting in to an order-1 page
+ * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
  */
-#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
-				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
-#define KMALLOC_SHIFT_MAX	KMALLOC_SHIFT_HIGH
+#ifdef CONFIG_SLAB
+#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
+#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)
 #ifndef KMALLOC_SHIFT_LOW
 #define KMALLOC_SHIFT_LOW	5
 #endif
 #endif
 
 #ifdef CONFIG_SLUB
-/*
- * SLUB directly allocates requests fitting in to an order-1 page
- * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
- */
 #define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
 #define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)
 #ifndef KMALLOC_SHIFT_LOW
@@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl
  *	Try really hard to succeed the allocation but fail
  *	eventually.
  */
+#ifndef CONFIG_SLOB
 static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 {
 	if (__builtin_constant_p(size)) {
-#ifndef CONFIG_SLOB
 		unsigned int index;
-#endif
+
 		if (size > KMALLOC_MAX_CACHE_SIZE)
 			return kmalloc_large(size, flags);
-#ifndef CONFIG_SLOB
+
 		index = kmalloc_index(size);
 
 		if (!index)
@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 		return kmem_cache_alloc_trace(
 				kmalloc_caches[kmalloc_type(flags)][index],
 				flags, size);
-#endif
 	}
 	return __kmalloc(size, flags);
 }
+#else
+static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
+{
+	if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)
+		return kmalloc_large(size, flags);
+	return __kmalloc(size, flags);
+}
+#endif
 
 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
mm/slab.c +27 -4
index ddf5737c63d9..570af6dc3478 100644
@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
 	void *ret;
 
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
 	void *ret;
 
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
 {
 	struct kmem_cache *s;
 	size_t i;
+	struct folio *folio;
 
 	local_irq_disable();
 	for (i = 0; i < size; i++) {
 		void *objp = p[i];
 
-		if (!orig_s) /* called via kfree_bulk */
+		if (!orig_s) {
+			/* called via kfree_bulk */
+			folio = virt_to_folio(objp);
+			if (unlikely(!folio_test_slab(folio))) {
+				free_large_kmalloc(folio, objp);
+				continue;
+			}
 			s = virt_to_cache(objp);
-		else
+		} else
 			s = cache_from_obj(orig_s, objp);
 		if (!s)
 			continue;
@@ -3776,11 +3785,20 @@ void kfree(const void *objp)
 {
 	struct kmem_cache *c;
 	unsigned long flags;
+	struct folio *folio;
+	void *object = (void *) objp;
 
 	trace_kfree(_RET_IP_, objp);
 
 	if (unlikely(ZERO_OR_NULL_PTR(objp)))
 		return;
+
+	folio = virt_to_folio(objp);
+	if (unlikely(!folio_test_slab(folio))) {
+		free_large_kmalloc(folio, object);
+		return;
+	}
+
 	local_irq_save(flags);
 	kfree_debugcheck(objp);
 	c = virt_to_cache(objp);
@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,
 size_t __ksize(const void *objp)
 {
 	struct kmem_cache *c;
+	struct folio *folio;
 	size_t size;
 
 	BUG_ON(!objp);
 	if (unlikely(objp == ZERO_SIZE_PTR))
 		return 0;
 
+	folio = virt_to_folio(objp);
+	if (!folio_test_slab(folio))
+		return folio_size(folio);
+
 	c = virt_to_cache(objp);
 	size = c ? c->object_size : 0;
 
mm/slab.h +19 -0
index c7f2abc2b154..31e98beb47a3 100644
@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
 		print_tracking(cachep, x);
 	return cachep;
 }
+
+static __always_inline void kfree_hook(void *x)
+{
+	kmemleak_free(x);
+	kasan_kfree_large(x);
+}
+
+static inline void free_large_kmalloc(struct folio *folio, void *object)
+{
+	unsigned int order = folio_order(folio);
+
+	if (WARN_ON_ONCE(order == 0))
+		pr_warn_once("object pointer: 0x%p\n", object);
+
+	kfree_hook(object);
+	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
+			      -(PAGE_SIZE << order));
+	__free_pages(folio_page(folio, 0), order);
+}
 #endif /* CONFIG_SLOB */
 
 static inline size_t slab_ksize(const struct kmem_cache *s)
mm/slub.c +0 -19
index 261474092e43..04fd084f4709 100644
@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
 	return ptr;
 }
 
-static __always_inline void kfree_hook(void *x)
-{
-	kmemleak_free(x);
-	kasan_kfree_large(x);
-}
-
 static __always_inline bool slab_free_hook(struct kmem_cache *s,
 						void *x, bool init)
 {
@@ -3535,19 +3529,6 @@ struct detached_freelist {
 	struct kmem_cache *s;
 };
 
-static inline void free_large_kmalloc(struct folio *folio, void *object)
-{
-	unsigned int order = folio_order(folio);
-
-	if (WARN_ON_ONCE(order == 0))
-		pr_warn_once("object pointer: 0x%p\n", object);
-
-	kfree_hook(object);
-	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
-			      -(PAGE_SIZE << order));
-	__free_pages(folio_page(folio, 0), order);
-}
-
 /*
  * This function progressively scans the array with free objects (with
  * a limited look ahead) and extract objects belonging to the same
Signature · 1 line
2.33.1
#2 lore raw

Re: [PATCH v2 1/5] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator

Quoted 5 lines · Hyeonggon Yoo

There is not much benefit for serving large objects in kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.

Looks like it's on the right path! Some suggestions follow.

Quoted 23 lines
---
 include/linux/slab.h | 35 ++++++++++++++++-------------------
 mm/slab.c            | 31 +++++++++++++++++++++++++++----
 mm/slab.h            | 19 +++++++++++++++++++
 mm/slub.c            | 19 -------------------
 4 files changed, 62 insertions(+), 42 deletions(-)
include/linux/slab.h +0 -7
index 37bde99b74af..e7b3330db4f3 100644
@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);
  * Kmalloc array related definitions
  */
 
-#ifdef CONFIG_SLAB
 /*
- * The largest kmalloc size supported by the SLAB allocators is
- * 32 megabyte (2^25) or the maximum allocatable page order if that is
- * less than 32 MB.
- *
- * WARNING: Its not easy to increase this value since the allocators have
- * to do various tricks to work around compiler limitations in order to

include/linux/slab.h void kmem_dump_obj(void *object); · line 233

Quoted 10 lines
include/linux/slab.h +5 -4
- * ensure proper constant folding.
+ * SLAB and SLUB directly allocates requests fitting in to an order-1 page
+ * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
  */
-#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
-				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
-#define KMALLOC_SHIFT_MAX	KMALLOC_SHIFT_HIGH
+#ifdef CONFIG_SLAB
+#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
+#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)

I think we should be able to also remove the larger sizes from __kmalloc_index() later in this file.

Quoted 56 lines
include/linux/slab.h +11 -8
 #ifndef KMALLOC_SHIFT_LOW
 #define KMALLOC_SHIFT_LOW	5
 #endif
 #endif
 
 #ifdef CONFIG_SLUB
-/*
- * SLUB directly allocates requests fitting in to an order-1 page
- * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
- */
 #define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
 #define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)
 #ifndef KMALLOC_SHIFT_LOW
@@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl
  *	Try really hard to succeed the allocation but fail
  *	eventually.
  */
+#ifndef CONFIG_SLOB
 static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 {
 	if (__builtin_constant_p(size)) {
-#ifndef CONFIG_SLOB
 		unsigned int index;
-#endif
+
 		if (size > KMALLOC_MAX_CACHE_SIZE)
 			return kmalloc_large(size, flags);
-#ifndef CONFIG_SLOB
+
 		index = kmalloc_index(size);
 
 		if (!index)
@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 		return kmem_cache_alloc_trace(
 				kmalloc_caches[kmalloc_type(flags)][index],
 				flags, size);
-#endif
 	}
 	return __kmalloc(size, flags);
 }
+#else
+static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
+{
+	if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)
+		return kmalloc_large(size, flags);
+	return __kmalloc(size, flags);
+}
+#endif
 
 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
mm/slab.c +0 -0
index ddf5737c63d9..570af6dc3478 100644

mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3627

Quoted 5 lines
mm/slab.c +1 -1
@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
 	void *ret;
 
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);

This loses the node information and NUMA locality. It would be better to generalize kmalloc_large_node() from mm/slub.c instead.

Quoted 20 lines
mm/slab.c +4 -1
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
 	void *ret;
 
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
 {
 	struct kmem_cache *s;
 	size_t i;
+	struct folio *folio;
 

mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3754

Quoted 10 lines
mm/slab.c +5 -1
 	local_irq_disable();
 	for (i = 0; i < size; i++) {
 		void *objp = p[i];
 
-		if (!orig_s) /* called via kfree_bulk */
+		if (!orig_s) {
+			/* called via kfree_bulk */
+			folio = virt_to_folio(objp);
+			if (unlikely(!folio_test_slab(folio))) {
+				free_large_kmalloc(folio, objp);

Hmm I'm not sure it's a good idea to call this and the page allocator with disabled irq's. Maybe simply re-enable around it?

mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3757

Quoted 3 lines
mm/slab.c +2 -0
+				continue;
+			}
 			s = virt_to_cache(objp);

Since we already have the folio, we shoold instead use folio_slab and slab->slab_cache (see how mm/slub.c does it).
The same applies to functions below as well.

Quoted 49 lines
mm/slab.c +15 -1
-		else
+		} else
 			s = cache_from_obj(orig_s, objp);
 		if (!s)
 			continue;
@@ -3776,11 +3785,20 @@ void kfree(const void *objp)
 {
 	struct kmem_cache *c;
 	unsigned long flags;
+	struct folio *folio;
+	void *object = (void *) objp;
 
 	trace_kfree(_RET_IP_, objp);
 
 	if (unlikely(ZERO_OR_NULL_PTR(objp)))
 		return;
+
+	folio = virt_to_folio(objp);
+	if (unlikely(!folio_test_slab(folio))) {
+		free_large_kmalloc(folio, object);
+		return;
+	}
+
 	local_irq_save(flags);
 	kfree_debugcheck(objp);
 	c = virt_to_cache(objp);
@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,
 size_t __ksize(const void *objp)
 {
 	struct kmem_cache *c;
+	struct folio *folio;
 	size_t size;
 
 	BUG_ON(!objp);
 	if (unlikely(objp == ZERO_SIZE_PTR))
 		return 0;
 
+	folio = virt_to_folio(objp);
+	if (!folio_test_slab(folio))
+		return folio_size(folio);
+
 	c = virt_to_cache(objp);
 	size = c ? c->object_size : 0;
 
mm/slab.h +0 -0
index c7f2abc2b154..31e98beb47a3 100644

mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 671

Quoted 8 lines
mm/slab.h +5 -0
@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
 		print_tracking(cachep, x);
 	return cachep;
 }
+
+static __always_inline void kfree_hook(void *x)
+{
+	kmemleak_free(x);
+	kasan_kfree_large(x);

Looks like there's only one caller of kfree_hook() so we could do that directly there.

Quoted 54 lines
mm/slab.h +14 -0
+}
+
+static inline void free_large_kmalloc(struct folio *folio, void *object)
+{
+	unsigned int order = folio_order(folio);
+
+	if (WARN_ON_ONCE(order == 0))
+		pr_warn_once("object pointer: 0x%p\n", object);
+
+	kfree_hook(object);
+	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
+			      -(PAGE_SIZE << order));
+	__free_pages(folio_page(folio, 0), order);
+}
 #endif /* CONFIG_SLOB */
 
 static inline size_t slab_ksize(const struct kmem_cache *s)
mm/slub.c +0 -19
index 261474092e43..04fd084f4709 100644
@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
 	return ptr;
 }
 
-static __always_inline void kfree_hook(void *x)
-{
-	kmemleak_free(x);
-	kasan_kfree_large(x);
-}
-
 static __always_inline bool slab_free_hook(struct kmem_cache *s,
 						void *x, bool init)
 {
@@ -3535,19 +3529,6 @@ struct detached_freelist {
 	struct kmem_cache *s;
 };
 
-static inline void free_large_kmalloc(struct folio *folio, void *object)
-{
-	unsigned int order = folio_order(folio);
-
-	if (WARN_ON_ONCE(order == 0))
-		pr_warn_once("object pointer: 0x%p\n", object);
-
-	kfree_hook(object);
-	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
-			      -(PAGE_SIZE << order));
-	__free_pages(folio_page(folio, 0), order);
-}
-
 /*
  * This function progressively scans the array with free objects (with
  * a limited look ahead) and extract objects belonging to the same
#3 lore raw

Re: [PATCH v2 1/5] mm/slab: kmalloc: pass requests larger than order-1 page to page allocator

Quoted 9 lines · Vlastimil Babka
Quoted 5 lines · Hyeonggon Yoo

There is not much benefit for serving large objects in kmalloc(). Let's pass large requests to page allocator like SLUB for better maintenance of common code.

Looks like it's on the right path! Some suggestions follow.

Thank you for reviewing this!
You have really good eye.

Quoted 28 lines
Quoted 28 lines
---
 include/linux/slab.h | 35 ++++++++++++++++-------------------
 mm/slab.c            | 31 +++++++++++++++++++++++++++----
 mm/slab.h            | 19 +++++++++++++++++++
 mm/slub.c            | 19 -------------------
 4 files changed, 62 insertions(+), 42 deletions(-)
include/linux/slab.h +2 -9
index 37bde99b74af..e7b3330db4f3 100644
@@ -224,29 +224,19 @@ void kmem_dump_obj(void *object);
  * Kmalloc array related definitions
  */
 
-#ifdef CONFIG_SLAB
 /*
- * The largest kmalloc size supported by the SLAB allocators is
- * 32 megabyte (2^25) or the maximum allocatable page order if that is
- * less than 32 MB.
- *
- * WARNING: Its not easy to increase this value since the allocators have
- * to do various tricks to work around compiler limitations in order to
- * ensure proper constant folding.
+ * SLAB and SLUB directly allocates requests fitting in to an order-1 page
+ * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
  */
-#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
Quoted 9 lines

include/linux/slab.h void kmem_dump_obj(void *object); · line 233

Quoted 5 lines
include/linux/slab.h +3 -2
-				(MAX_ORDER + PAGE_SHIFT - 1) : 25)
-#define KMALLOC_SHIFT_MAX	KMALLOC_SHIFT_HIGH
+#ifdef CONFIG_SLAB
+#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
+#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)

I think we should be able to also remove the larger sizes from __kmalloc_index() later in this file.

Right. we can remove sizes larger than max(PAGE_SHIFT) + 1.

searching using cscope, maximum PAGE_SHIFT is 20 (1MB) in hexagon. I think we can remove anything larger than 2MB.

But Hmm, I'll add something like static_assert(PAGE_SHIFT <= 20). Let's see if any architecture/config complain this.

Quoted 58 lines
Quoted 58 lines
include/linux/slab.h +11 -8
 #ifndef KMALLOC_SHIFT_LOW
 #define KMALLOC_SHIFT_LOW	5
 #endif
 #endif
 
 #ifdef CONFIG_SLUB
-/*
- * SLUB directly allocates requests fitting in to an order-1 page
- * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.
- */
 #define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)
 #define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)
 #ifndef KMALLOC_SHIFT_LOW
@@ -564,15 +554,15 @@ static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t fl
  *	Try really hard to succeed the allocation but fail
  *	eventually.
  */
+#ifndef CONFIG_SLOB
 static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 {
 	if (__builtin_constant_p(size)) {
-#ifndef CONFIG_SLOB
 		unsigned int index;
-#endif
+
 		if (size > KMALLOC_MAX_CACHE_SIZE)
 			return kmalloc_large(size, flags);
-#ifndef CONFIG_SLOB
+
 		index = kmalloc_index(size);
 
 		if (!index)
@@ -581,10 +571,17 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
 		return kmem_cache_alloc_trace(
 				kmalloc_caches[kmalloc_type(flags)][index],
 				flags, size);
-#endif
 	}
 	return __kmalloc(size, flags);
 }
+#else
+static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
+{
+	if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)
+		return kmalloc_large(size, flags);
+	return __kmalloc(size, flags);
+}
+#endif
 
 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
mm/slab.c +0 -0
index ddf5737c63d9..570af6dc3478 100644
@@ -3624,7 +3624,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
 	void *ret;
 
Quoted 5 lines

mm/slab.c __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) · line 3627

Quoted 3 lines
mm/slab.c +1 -1
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);

This loses the node information and NUMA locality.

Oh, my fault. Thank you for catching this.

Quoted 1 line

It would be better to generalize kmalloc_large_node() from mm/slub.c instead.

Sounds nice.

Quoted 24 lines
Quoted 24 lines
mm/slab.c +4 -1
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3685,7 +3686,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
 	void *ret;
 
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-		return NULL;
+		return kmalloc_large(size, flags);
+
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
@@ -3739,14 +3741,21 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
 {
 	struct kmem_cache *s;
 	size_t i;
+	struct folio *folio;
 
 	local_irq_disable();
 	for (i = 0; i < size; i++) {
 		void *objp = p[i];
 
Quoted 9 lines

mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3754

Quoted 6 lines
mm/slab.c +5 -1
-		if (!orig_s) /* called via kfree_bulk */
+		if (!orig_s) {
+			/* called via kfree_bulk */
+			folio = virt_to_folio(objp);
+			if (unlikely(!folio_test_slab(folio))) {
+				free_large_kmalloc(folio, objp);

Hmm I'm not sure it's a good idea to call this and the page allocator with disabled irq's. Maybe simply re-enable around it?

Seems reasonable. re-enabling around here will be sufficient for this function.

By The Way, it seems SLAB really does not care about disabling irq even if it's unnecessary.

I would like to try reducing irq-disabled area in SLAB and how it affects on macro/micro benchmarks after this series is done :)

Quoted 7 lines

mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) · line 3757

Quoted 3 lines
mm/slab.c +2 -0
+				continue;
+			}
 			s = virt_to_cache(objp);

Since we already have the folio, we shoold instead use folio_slab and slab->slab_cache (see how mm/slub.c does it).
The same applies to functions below as well.

Right. I'll change them in v3.

Quoted 44 lines
Quoted 44 lines
mm/slab.c +15 -1
-		else
+		} else
 			s = cache_from_obj(orig_s, objp);
 		if (!s)
 			continue;
@@ -3776,11 +3785,20 @@ void kfree(const void *objp)
 {
 	struct kmem_cache *c;
 	unsigned long flags;
+	struct folio *folio;
+	void *object = (void *) objp;
 
 	trace_kfree(_RET_IP_, objp);
 
 	if (unlikely(ZERO_OR_NULL_PTR(objp)))
 		return;
+
+	folio = virt_to_folio(objp);
+	if (unlikely(!folio_test_slab(folio))) {
+		free_large_kmalloc(folio, object);
+		return;
+	}
+
 	local_irq_save(flags);
 	kfree_debugcheck(objp);
 	c = virt_to_cache(objp);
@@ -4211,12 +4229,17 @@ void __check_heap_object(const void *ptr, unsigned long n,
 size_t __ksize(const void *objp)
 {
 	struct kmem_cache *c;
+	struct folio *folio;
 	size_t size;
 
 	BUG_ON(!objp);
 	if (unlikely(objp == ZERO_SIZE_PTR))
 		return 0;
 
+	folio = virt_to_folio(objp);
+	if (!folio_test_slab(folio))
+		return folio_size(folio);
+
 	c = virt_to_cache(objp);
 	size = c ? c->object_size : 0;
 
Quoted 17 lines

mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 671

Quoted 13 lines
mm/slab.h +5 -0
index c7f2abc2b154..31e98beb47a3 100644
@@ -664,6 +664,25 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
 		print_tracking(cachep, x);
 	return cachep;
 }
+
+static __always_inline void kfree_hook(void *x)
+{
+	kmemleak_free(x);
+	kasan_kfree_large(x);

Looks like there's only one caller of kfree_hook() so we could do that directly there.

Right.

Quoted 38 lines
Quoted 38 lines
mm/slab.h +14 -0
+}
+
+static inline void free_large_kmalloc(struct folio *folio, void *object)
+{
+	unsigned int order = folio_order(folio);
+
+	if (WARN_ON_ONCE(order == 0))
+		pr_warn_once("object pointer: 0x%p\n", object);
+
+	kfree_hook(object);
+	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
+			      -(PAGE_SIZE << order));
+	__free_pages(folio_page(folio, 0), order);
+}
 #endif /* CONFIG_SLOB */
 
 static inline size_t slab_ksize(const struct kmem_cache *s)
mm/slub.c +0 -6
index 261474092e43..04fd084f4709 100644
@@ -1686,12 +1686,6 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
 	return ptr;
 }
 
-static __always_inline void kfree_hook(void *x)
-{
-	kmemleak_free(x);
-	kasan_kfree_large(x);
-}
-
 static __always_inline bool slab_free_hook(struct kmem_cache *s,
 						void *x, bool init)
 {
@@ -3535,19 +3529,6 @@ struct detached_freelist {
 	struct kmem_cache *s;
 };
 
Quoted 17 lines
Quoted 6 lines
mm/slub.c +0 -6
-static inline void free_large_kmalloc(struct folio *folio, void *object)
-{
-	unsigned int order = folio_order(folio);
-
-	if (WARN_ON_ONCE(order == 0))
-		pr_warn_once("object pointer: 0x%p\n", object);

mm/slub.c struct detached_freelist { · line 3534

Quoted 10 lines
mm/slub.c +0 -7
-
-	kfree_hook(object);
-	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
-			      -(PAGE_SIZE << order));
-	__free_pages(folio_page(folio, 0), order);
-}
-
 /*
  * This function progressively scans the array with free objects (with
  * a limited look ahead) and extract objects belonging to the same

Thank you so much!

Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)
#4 lore raw

PATCH v2 2/5 mm/sl[au]b: unify __ksize()

In reply to Hyeonggon Yoo

Now that SLAB passes large requests to page allocator like SLUB, Unify __ksize(). Only SLOB need to implement own version of __ksize() because it stores size in object header for kmalloc objects.

---
 mm/slab.c        | 30 ------------------------------
 mm/slab_common.c | 27 +++++++++++++++++++++++++++
 mm/slub.c        | 16 ----------------
 3 files changed, 27 insertions(+), 46 deletions(-)
mm/slab.c +0 -30
index 570af6dc3478..3ddf2181d8e4 100644
@@ -4216,33 +4216,3 @@ void __check_heap_object(const void *ptr, unsigned long n,
 	usercopy_abort("SLAB object", cachep->name, to_user, offset, n);
 }
 #endif /* CONFIG_HARDENED_USERCOPY */
-
-/**
- * __ksize -- Uninstrumented ksize.
- * @objp: pointer to the object
- *
- * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
- * safety checks as ksize() with KASAN instrumentation enabled.
- *
- * Return: size of the actual memory used by @objp in bytes
- */
-size_t __ksize(const void *objp)
-{
-	struct kmem_cache *c;
-	struct folio *folio;
-	size_t size;
-
-	BUG_ON(!objp);
-	if (unlikely(objp == ZERO_SIZE_PTR))
-		return 0;
-
-	folio = virt_to_folio(objp);
-	if (!folio_test_slab(folio))
-		return folio_size(folio);
-
-	c = virt_to_cache(objp);
-	size = c ? c->object_size : 0;
-
-	return size;
-}
-EXPORT_SYMBOL(__ksize);
mm/slab_common.c +27 -0
index 23f2ab0713b7..1d2f92e871d2 100644
@@ -1245,6 +1245,33 @@ void kfree_sensitive(const void *p)
 }
 EXPORT_SYMBOL(kfree_sensitive);
 
+#ifndef CONFIG_SLOB
+/**
+ * __ksize -- Uninstrumented ksize.
+ * @objp: pointer to the object
+ *
+ * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
+ * safety checks as ksize() with KASAN instrumentation enabled.
+ *
+ * Return: size of the actual memory used by @objp in bytes
+ */
+size_t __ksize(const void *object)
+{
+	struct folio *folio;
+
+	if (unlikely(object == ZERO_SIZE_PTR))
+		return 0;
+
+	folio = virt_to_folio(object);
+
+	if (unlikely(!folio_test_slab(folio)))
+		return folio_size(folio);
+
+	return slab_ksize(folio_slab(folio)->slab_cache);
+}
+EXPORT_SYMBOL(__ksize);
+#endif
+
 /**
  * ksize - get the actual amount of memory allocated for a given object
  * @objp: Pointer to the object
mm/slub.c +0 -16
index 04fd084f4709..6f0ebadd8f30 100644
@@ -4507,22 +4507,6 @@ void __check_heap_object(const void *ptr, unsigned long n,
 }
 #endif /* CONFIG_HARDENED_USERCOPY */
 
-size_t __ksize(const void *object)
-{
-	struct folio *folio;
-
-	if (unlikely(object == ZERO_SIZE_PTR))
-		return 0;
-
-	folio = virt_to_folio(object);
-
-	if (unlikely(!folio_test_slab(folio)))
-		return folio_size(folio);
-
-	return slab_ksize(folio_slab(folio)->slab_cache);
-}
-EXPORT_SYMBOL(__ksize);
-
 void kfree(const void *x)
 {
 	struct folio *folio;
Signature · 1 line
2.33.1
Reviewed-by on v2 2/5
#5 lore raw

Re: [PATCH v2 2/5] mm/sl[au]b: unify __ksize()

Quoted 5 lines · Hyeonggon Yoo

Now that SLAB passes large requests to page allocator like SLUB, Unify __ksize(). Only SLOB need to implement own version of __ksize() because it stores size in object header for kmalloc objects.

As discussed, we can be more specific about the !folio_test_slab() case, but that can be done on top.

#6 lore raw

Re: [PATCH v2 2/5] mm/sl[au]b: unify __ksize()

Quoted 11 lines · Vlastimil Babka
Quoted 5 lines · Hyeonggon Yoo

Now that SLAB passes large requests to page allocator like SLUB, Unify __ksize(). Only SLOB need to implement own version of __ksize() because it stores size in object header for kmalloc objects.

As discussed, we can be more specific about the !folio_test_slab() case, but that can be done on top.

I'll add that in v3. Thanks!

Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)
#7 lore raw

PATCH v2 3/5 mm/sl[auo]b: move definition of __ksize() to mm/slab.h

In reply to Hyeonggon Yoo

__ksize() is only called by KASAN. Remove export symbol and move definition to mm/slab.h as we don't want to grow its callers.

[ [email protected]: Move definition to mm/slab.h and reduce comments ]

---
 include/linux/slab.h | 1 -
 mm/slab.h            | 2 ++
 mm/slab_common.c     | 9 +--------
 mm/slob.c            | 1 -
 4 files changed, 3 insertions(+), 10 deletions(-)
include/linux/slab.h +0 -1
index e7b3330db4f3..d2b896553315 100644
@@ -182,7 +182,6 @@ int kmem_cache_shrink(struct kmem_cache *s);
 void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __alloc_size(2);
 void kfree(const void *objp);
 void kfree_sensitive(const void *objp);
-size_t __ksize(const void *objp);
 size_t ksize(const void *objp);
 #ifdef CONFIG_PRINTK
 bool kmem_valid_obj(void *object);
mm/slab.h +2 -0
index 31e98beb47a3..79b319d58504 100644
@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object)
 }
 #endif /* CONFIG_SLOB */
 
+size_t __ksize(const void *objp);
+
 static inline size_t slab_ksize(const struct kmem_cache *s)
 {
 #ifndef CONFIG_SLUB
mm/slab_common.c +1 -8
index 1d2f92e871d2..b126fc7247b9 100644
@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);
 
 #ifndef CONFIG_SLOB
 /**
- * __ksize -- Uninstrumented ksize.
- * @objp: pointer to the object
- *
- * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
- * safety checks as ksize() with KASAN instrumentation enabled.
- *
- * Return: size of the actual memory used by @objp in bytes
+ * __ksize -- Uninstrumented ksize. Only called by KASAN.
  */
 size_t __ksize(const void *object)
 {
@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object)
 
 	return slab_ksize(folio_slab(folio)->slab_cache);
 }
-EXPORT_SYMBOL(__ksize);
 #endif
 
 /**
mm/slob.c +0 -1
index 60c5842215f1..d8af6c54f133 100644
@@ -588,7 +588,6 @@ size_t __ksize(const void *block)
 	m = (unsigned int *)(block - align);
 	return SLOB_UNITS(*m) * SLOB_UNIT;
 }
-EXPORT_SYMBOL(__ksize);
 
 int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)
 {
Signature · 1 line
2.33.1
Reviewed-by on v2 3/5
#8 lore raw

Re: [PATCH v2 3/5] mm/sl[auo]b: move definition of __ksize() to mm/slab.h

Quoted 6 lines · Hyeonggon Yoo

__ksize() is only called by KASAN. Remove export symbol and move definition to mm/slab.h as we don't want to grow its callers.

[ [email protected]: Move definition to mm/slab.h and reduce comments ]

Quoted 16 lines
mm/slab.h +2 -0
@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object)
 }
 #endif /* CONFIG_SLOB */
 
+size_t __ksize(const void *objp);
+
 static inline size_t slab_ksize(const struct kmem_cache *s)
 {
 #ifndef CONFIG_SLUB
mm/slab_common.c +0 -0
index 1d2f92e871d2..b126fc7247b9 100644

mm/slab_common.c EXPORT_SYMBOL(kfree_sensitive); · line 1249

Quoted 3 lines
mm/slab_common.c +0 -0
@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);
 
 #ifndef CONFIG_SLOB
 /**

Maybe just /* so it's not even parsed as a kernel-doc anymore?

Quoted 18 lines
mm/slab_common.c +1 -8
- * __ksize -- Uninstrumented ksize.
- * @objp: pointer to the object
- *
- * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
- * safety checks as ksize() with KASAN instrumentation enabled.
- *
- * Return: size of the actual memory used by @objp in bytes
+ * __ksize -- Uninstrumented ksize. Only called by KASAN.
  */
 size_t __ksize(const void *object)
 {
@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object)
 
 	return slab_ksize(folio_slab(folio)->slab_cache);
 }
-EXPORT_SYMBOL(__ksize);
 #endif
 
#9 lore raw

Re: [PATCH v2 3/5] mm/sl[auo]b: move definition of __ksize() to mm/slab.h

Quoted 10 lines · Vlastimil Babka
Quoted 6 lines · Hyeonggon Yoo

__ksize() is only called by KASAN. Remove export symbol and move definition to mm/slab.h as we don't want to grow its callers.

[ [email protected]: Move definition to mm/slab.h and reduce comments ]

Thanks!

Quoted 17 lines
Quoted 17 lines
mm/slab.h +2 -0
@@ -685,6 +685,8 @@ static inline void free_large_kmalloc(struct folio *folio, void *object)
 }
 #endif /* CONFIG_SLOB */
 
+size_t __ksize(const void *objp);
+
 static inline size_t slab_ksize(const struct kmem_cache *s)
 {
 #ifndef CONFIG_SLUB
mm/slab_common.c +0 -0
index 1d2f92e871d2..b126fc7247b9 100644
@@ -1247,13 +1247,7 @@ EXPORT_SYMBOL(kfree_sensitive);
 
Quoted 5 lines

mm/slab_common.c EXPORT_SYMBOL(kfree_sensitive); · line 1249

Quoted 2 lines
mm/slab_common.c +0 -0
 #ifndef CONFIG_SLOB
 /**

Maybe just /* so it's not even parsed as a kernel-doc anymore?

Oh yes, that would be better.

Quoted 18 lines
Quoted 18 lines
mm/slab_common.c +1 -8
- * __ksize -- Uninstrumented ksize.
- * @objp: pointer to the object
- *
- * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
- * safety checks as ksize() with KASAN instrumentation enabled.
- *
- * Return: size of the actual memory used by @objp in bytes
+ * __ksize -- Uninstrumented ksize. Only called by KASAN.
  */
 size_t __ksize(const void *object)
 {
@@ -1269,7 +1263,6 @@ size_t __ksize(const void *object)
 
 	return slab_ksize(folio_slab(folio)->slab_cache);
 }
-EXPORT_SYMBOL(__ksize);
 #endif
 
Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)
#10 lore raw

PATCH v2 4/5 mm/slub: limit number of node partial slabs only in cache creation

In reply to Hyeonggon Yoo

SLUB sets number of minimum partial slabs for node (min_partial) using set_min_partial(). SLUB holds at least min_partial slabs even if they're empty to avoid excessive use of page allocator.

set_min_partial() limits value of min_partial limits value of min_partial MIN_PARTIAL and MAX_PARTIAL. As set_min_partial() can be called by min_partial_store() too, Only limit value of min_partial in kmem_cache_open() so that it can be changed to value that a user wants.

[ [email protected]: Fold set_min_partial() into its callers ]

---
 mm/slub.c | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)
mm/slub.c +3 -11
index 6f0ebadd8f30..f9ae983a3dc6 100644
@@ -3981,15 +3981,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
 	return 1;
 }
 
-static void set_min_partial(struct kmem_cache *s, unsigned long min)
-{
-	if (min < MIN_PARTIAL)
-		min = MIN_PARTIAL;
-	else if (min > MAX_PARTIAL)
-		min = MAX_PARTIAL;
-	s->min_partial = min;
-}
-
 static void set_cpu_partial(struct kmem_cache *s)
 {
 #ifdef CONFIG_SLUB_CPU_PARTIAL
@@ -4196,7 +4187,8 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
 	 * The larger the object size is, the more slabs we want on the partial
 	 * list to avoid pounding the page allocator excessively.
 	 */
-	set_min_partial(s, ilog2(s->size) / 2);
+	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
+	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
 
 	set_cpu_partial(s);
 
@@ -5361,7 +5353,7 @@ static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
 	if (err)
 		return err;
 
-	set_min_partial(s, min);
+	s->min_partial = min;
 	return length;
 }
 SLAB_ATTR(min_partial);
Signature · 1 line
2.33.1
Reviewed-by on v2 4/5
#11 lore raw

Re: [PATCH v2 4/5] mm/slub: limit number of node partial slabs only in cache creation

Quoted 4 lines · Hyeonggon Yoo

SLUB sets number of minimum partial slabs for node (min_partial) using set_min_partial(). SLUB holds at least min_partial slabs even if they're empty to avoid excessive use of page allocator.

Quoted 8 lines

set_min_partial() limits value of min_partial limits value of min_partial MIN_PARTIAL and MAX_PARTIAL. As set_min_partial() can be called by min_partial_store() too, Only limit value of min_partial in kmem_cache_open() so that it can be changed to value that a user wants.

[ [email protected]: Fold set_min_partial() into its callers ]

Quoted 43 lines
---
 mm/slub.c | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)
mm/slub.c +3 -11
index 6f0ebadd8f30..f9ae983a3dc6 100644
@@ -3981,15 +3981,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
 	return 1;
 }
 
-static void set_min_partial(struct kmem_cache *s, unsigned long min)
-{
-	if (min < MIN_PARTIAL)
-		min = MIN_PARTIAL;
-	else if (min > MAX_PARTIAL)
-		min = MAX_PARTIAL;
-	s->min_partial = min;
-}
-
 static void set_cpu_partial(struct kmem_cache *s)
 {
 #ifdef CONFIG_SLUB_CPU_PARTIAL
@@ -4196,7 +4187,8 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
 	 * The larger the object size is, the more slabs we want on the partial
 	 * list to avoid pounding the page allocator excessively.
 	 */
-	set_min_partial(s, ilog2(s->size) / 2);
+	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
+	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
 
 	set_cpu_partial(s);
 
@@ -5361,7 +5353,7 @@ static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
 	if (err)
 		return err;
 
-	set_min_partial(s, min);
+	s->min_partial = min;
 	return length;
 }
 SLAB_ATTR(min_partial);
#12 lore raw

PATCH v2 5/5 mm/slub: refactor deactivate_slab()

In reply to Hyeonggon Yoo

Simplify deactivate_slab() by unlocking n->list_lock and retrying cmpxchg_double() when cmpxchg_double() fails, and perform add_{partial,full} only when it succeed.

Releasing and taking n->list_lock again here is not harmful as SLUB avoids deactivating slabs as much as possible.

[ [email protected]: perform add_{partial,full} when cmpxchg_double()
succeed. ]

---
 mm/slub.c | 81 ++++++++++++++++++++++---------------------------------
 1 file changed, 32 insertions(+), 49 deletions(-)
mm/slub.c +32 -49
index f9ae983a3dc6..c1a693ec5874 100644
@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 {
 	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
-	int lock = 0, free_delta = 0;
-	enum slab_modes l = M_NONE, m = M_NONE;
+	int free_delta = 0;
+	enum slab_modes mode = M_NONE;
 	void *nextfree, *freelist_iter, *freelist_tail;
 	int tail = DEACTIVATE_TO_HEAD;
 	unsigned long flags = 0;
@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	 * Ensure that the slab is unfrozen while the list presence
 	 * reflects the actual number of objects during unfreeze.
 	 *
-	 * We setup the list membership and then perform a cmpxchg
-	 * with the count. If there is a mismatch then the slab
-	 * is not unfrozen but the slab is on the wrong list.
-	 *
-	 * Then we restart the process which may have to remove
-	 * the slab from the list that we just put it on again
-	 * because the number of objects in the slab may have
-	 * changed.
+	 * We first perform cmpxchg holding lock and insert to list
+	 * when it succeed. If there is mismatch then slub is not
+	 * unfrozen and number of objects in the slab may have changed.
+	 * Then release lock and retry cmpxchg again.
 	 */
 redo:
 
@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	new.frozen = 0;
 
 	if (!new.inuse && n->nr_partial >= s->min_partial)
-		m = M_FREE;
+		mode = M_FREE;
 	else if (new.freelist) {
-		m = M_PARTIAL;
-		if (!lock) {
-			lock = 1;
-			/*
-			 * Taking the spinlock removes the possibility that
-			 * acquire_slab() will see a slab that is frozen
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
-	} else {
-		m = M_FULL;
-		if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {
-			lock = 1;
-			/*
-			 * This also ensures that the scanning of full
-			 * slabs from diagnostic functions will not see
-			 * any frozen slabs.
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
+		mode = M_PARTIAL;
+		/*
+		 * Taking the spinlock removes the possibility that
+		 * acquire_slab() will see a slab that is frozen
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
+	} else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
+		mode = M_FULL;
+		/*
+		 * This also ensures that the scanning of full
+		 * slabs from diagnostic functions will not see
+		 * any frozen slabs.
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
 	}
 
-	if (l != m) {
-		if (l == M_PARTIAL)
-			remove_partial(n, slab);
-		else if (l == M_FULL)
-			remove_full(s, n, slab);
-
-		if (m == M_PARTIAL)
-			add_partial(n, slab, tail);
-		else if (m == M_FULL)
-			add_full(s, n, slab);
-	}
 
-	l = m;
 	if (!cmpxchg_double_slab(s, slab,
 				old.freelist, old.counters,
 				new.freelist, new.counters,
-				"unfreezing slab"))
+				"unfreezing slab")) {
+		if (mode == M_PARTIAL || mode == M_FULL)
+			spin_unlock_irqrestore(&n->list_lock, flags);
 		goto redo;
+	}
 
-	if (lock)
-		spin_unlock_irqrestore(&n->list_lock, flags);
 
-	if (m == M_PARTIAL)
+	if (mode == M_PARTIAL) {
+		add_partial(n, slab, tail);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, tail);
-	else if (m == M_FULL)
+	} else if (mode == M_FULL) {
+		add_full(s, n, slab);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, DEACTIVATE_FULL);
-	else if (m == M_FREE) {
+	} else if (mode == M_FREE) {
 		stat(s, DEACTIVATE_EMPTY);
 		discard_slab(s, slab);
 		stat(s, FREE_SLAB);
Signature · 1 line
2.33.1
#13 lore raw

Re: [PATCH v2 5/5] mm/slub: refactor deactivate_slab()

Quoted 11 lines · Hyeonggon Yoo

Simplify deactivate_slab() by unlocking n->list_lock and retrying cmpxchg_double() when cmpxchg_double() fails, and perform add_{partial,full} only when it succeed.

Releasing and taking n->list_lock again here is not harmful as SLUB avoids deactivating slabs as much as possible.

[ [email protected]: perform add_{partial,full} when cmpxchg_double()
succeed. ]

Looks good, just noticed a tiny issue.

Quoted 47 lines
---
 mm/slub.c | 81 ++++++++++++++++++++++---------------------------------
 1 file changed, 32 insertions(+), 49 deletions(-)
mm/slub.c +7 -13
index f9ae983a3dc6..c1a693ec5874 100644
@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 {
 	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
-	int lock = 0, free_delta = 0;
-	enum slab_modes l = M_NONE, m = M_NONE;
+	int free_delta = 0;
+	enum slab_modes mode = M_NONE;
 	void *nextfree, *freelist_iter, *freelist_tail;
 	int tail = DEACTIVATE_TO_HEAD;
 	unsigned long flags = 0;
@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	 * Ensure that the slab is unfrozen while the list presence
 	 * reflects the actual number of objects during unfreeze.
 	 *
-	 * We setup the list membership and then perform a cmpxchg
-	 * with the count. If there is a mismatch then the slab
-	 * is not unfrozen but the slab is on the wrong list.
-	 *
-	 * Then we restart the process which may have to remove
-	 * the slab from the list that we just put it on again
-	 * because the number of objects in the slab may have
-	 * changed.
+	 * We first perform cmpxchg holding lock and insert to list
+	 * when it succeed. If there is mismatch then slub is not
+	 * unfrozen and number of objects in the slab may have changed.
+	 * Then release lock and retry cmpxchg again.
 	 */
 redo:
 
@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	new.frozen = 0;
 
 	if (!new.inuse && n->nr_partial >= s->min_partial)
-		m = M_FREE;
+		mode = M_FREE;
 	else if (new.freelist) {
-		m = M_PARTIAL;
-		if (!lock) {

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2430

Quoted 10 lines
mm/slub.c +0 -10
-			lock = 1;
-			/*
-			 * Taking the spinlock removes the possibility that
-			 * acquire_slab() will see a slab that is frozen
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
-	} else {
-		m = M_FULL;
-		if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {

This used to set m = M_FULL; always.

Quoted 6 lines
mm/slub.c +0 -6
-			lock = 1;
-			/*
-			 * This also ensures that the scanning of full
-			 * slabs from diagnostic functions will not see
-			 * any frozen slabs.
-			 */

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2422

Quoted 10 lines
mm/slub.c +8 -2
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
+		mode = M_PARTIAL;
+		/*
+		 * Taking the spinlock removes the possibility that
+		 * acquire_slab() will see a slab that is frozen
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
+	} else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
+		mode = M_FULL;

Now you only set it for SLAB_STORE_USER caches.

Quoted 34 lines
mm/slub.c +10 -15
+		/*
+		 * This also ensures that the scanning of full
+		 * slabs from diagnostic functions will not see
+		 * any frozen slabs.
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
 	}
 
-	if (l != m) {
-		if (l == M_PARTIAL)
-			remove_partial(n, slab);
-		else if (l == M_FULL)
-			remove_full(s, n, slab);
-
-		if (m == M_PARTIAL)
-			add_partial(n, slab, tail);
-		else if (m == M_FULL)
-			add_full(s, n, slab);
-	}
 
-	l = m;
 	if (!cmpxchg_double_slab(s, slab,
 				old.freelist, old.counters,
 				new.freelist, new.counters,
-				"unfreezing slab"))
+				"unfreezing slab")) {
+		if (mode == M_PARTIAL || mode == M_FULL)
+			spin_unlock_irqrestore(&n->list_lock, flags);
 		goto redo;
+	}
 
-	if (lock)
-		spin_unlock_irqrestore(&n->list_lock, flags);
 

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2449

Quoted 10 lines
mm/slub.c +6 -2
-	if (m == M_PARTIAL)
+	if (mode == M_PARTIAL) {
+		add_partial(n, slab, tail);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, tail);
-	else if (m == M_FULL)
+	} else if (mode == M_FULL) {
+		add_full(s, n, slab);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, DEACTIVATE_FULL);

As a result, full slabs without SLAB_STORE_USER will not count DEACTIVATE_FULL anymore.
I guess the easiest way to solve it is to e.g. add a M_FULL_NOLIST mode that only does the DEACTIVATE_FULL counting.

Quoted 5 lines
mm/slub.c +1 -1
-	else if (m == M_FREE) {
+	} else if (mode == M_FREE) {
 		stat(s, DEACTIVATE_EMPTY);
 		discard_slab(s, slab);
 		stat(s, FREE_SLAB);
#14 lore raw

Re: [PATCH v2 5/5] mm/slub: refactor deactivate_slab()

Quoted 128 lines · Vlastimil Babka
Quoted 11 lines · Hyeonggon Yoo

Simplify deactivate_slab() by unlocking n->list_lock and retrying cmpxchg_double() when cmpxchg_double() fails, and perform add_{partial,full} only when it succeed.

Releasing and taking n->list_lock again here is not harmful as SLUB avoids deactivating slabs as much as possible.

[ [email protected]: perform add_{partial,full} when cmpxchg_double()
succeed. ]

Looks good, just noticed a tiny issue.

Quoted 47 lines
---
 mm/slub.c | 81 ++++++++++++++++++++++---------------------------------
 1 file changed, 32 insertions(+), 49 deletions(-)
mm/slub.c +7 -13
index f9ae983a3dc6..c1a693ec5874 100644
@@ -2344,8 +2344,8 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 {
 	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
-	int lock = 0, free_delta = 0;
-	enum slab_modes l = M_NONE, m = M_NONE;
+	int free_delta = 0;
+	enum slab_modes mode = M_NONE;
 	void *nextfree, *freelist_iter, *freelist_tail;
 	int tail = DEACTIVATE_TO_HEAD;
 	unsigned long flags = 0;
@@ -2387,14 +2387,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	 * Ensure that the slab is unfrozen while the list presence
 	 * reflects the actual number of objects during unfreeze.
 	 *
-	 * We setup the list membership and then perform a cmpxchg
-	 * with the count. If there is a mismatch then the slab
-	 * is not unfrozen but the slab is on the wrong list.
-	 *
-	 * Then we restart the process which may have to remove
-	 * the slab from the list that we just put it on again
-	 * because the number of objects in the slab may have
-	 * changed.
+	 * We first perform cmpxchg holding lock and insert to list
+	 * when it succeed. If there is mismatch then slub is not
+	 * unfrozen and number of objects in the slab may have changed.
+	 * Then release lock and retry cmpxchg again.
 	 */
 redo:
 
@@ -2414,57 +2410,44 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	new.frozen = 0;
 
 	if (!new.inuse && n->nr_partial >= s->min_partial)
-		m = M_FREE;
+		mode = M_FREE;
 	else if (new.freelist) {
-		m = M_PARTIAL;
-		if (!lock) {

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2430

Quoted 10 lines
mm/slub.c +0 -10
-			lock = 1;
-			/*
-			 * Taking the spinlock removes the possibility that
-			 * acquire_slab() will see a slab that is frozen
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
-	} else {
-		m = M_FULL;
-		if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {

This used to set m = M_FULL; always.

Quoted 6 lines
mm/slub.c +0 -6
-			lock = 1;
-			/*
-			 * This also ensures that the scanning of full
-			 * slabs from diagnostic functions will not see
-			 * any frozen slabs.
-			 */

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2422

Quoted 10 lines
mm/slub.c +8 -2
-			spin_lock_irqsave(&n->list_lock, flags);
-		}
+		mode = M_PARTIAL;
+		/*
+		 * Taking the spinlock removes the possibility that
+		 * acquire_slab() will see a slab that is frozen
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
+	} else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
+		mode = M_FULL;

Now you only set it for SLAB_STORE_USER caches.

Quoted 34 lines
mm/slub.c +10 -15
+		/*
+		 * This also ensures that the scanning of full
+		 * slabs from diagnostic functions will not see
+		 * any frozen slabs.
+		 */
+		spin_lock_irqsave(&n->list_lock, flags);
 	}
 
-	if (l != m) {
-		if (l == M_PARTIAL)
-			remove_partial(n, slab);
-		else if (l == M_FULL)
-			remove_full(s, n, slab);
-
-		if (m == M_PARTIAL)
-			add_partial(n, slab, tail);
-		else if (m == M_FULL)
-			add_full(s, n, slab);
-	}
 
-	l = m;
 	if (!cmpxchg_double_slab(s, slab,
 				old.freelist, old.counters,
 				new.freelist, new.counters,
-				"unfreezing slab"))
+				"unfreezing slab")) {
+		if (mode == M_PARTIAL || mode == M_FULL)
+			spin_unlock_irqrestore(&n->list_lock, flags);
 		goto redo;
+	}
 
-	if (lock)
-		spin_unlock_irqrestore(&n->list_lock, flags);
 
Quoted 13 lines

mm/slub.c static void deactivate_slab(struct kmem_cache *s, struct slab *slab, · line 2449

Quoted 10 lines
mm/slub.c +6 -2
-	if (m == M_PARTIAL)
+	if (mode == M_PARTIAL) {
+		add_partial(n, slab, tail);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, tail);
-	else if (m == M_FULL)
+	} else if (mode == M_FULL) {
+		add_full(s, n, slab);
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, DEACTIVATE_FULL);

As a result, full slabs without SLAB_STORE_USER will not count DEACTIVATE_FULL anymore.

Oh, thank you for catching this.

We usually only deactivate full slabs for debugging to track all list of slabs.

But as you pointed, in rare case when pfmemalloc flag does not match, full slabs can be deactivated (even if they are not put on list).

Quoted 2 lines

I guess the easiest way to solve it is to e.g. add a M_FULL_NOLIST mode that only does the DEACTIVATE_FULL counting.

That will be enough solution.

Will fix this in v3. Thank you!

Quoted 6 lines
Quoted 5 lines
mm/slub.c +1 -1
-	else if (m == M_FREE) {
+	} else if (mode == M_FREE) {
 		stat(s, DEACTIVATE_EMPTY);
 		discard_slab(s, slab);
 		stat(s, FREE_SLAB);
Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)
#15 lore raw
In reply to Hyeonggon Yoo
Quoted 11 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Quoted 25 lines

Hello, these are cleanup patches for slab.
Please consider them for slab-next :)

Any comments will be appreciated.
Thanks.

Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify __ksize()
mm/sl[auo]b: move definition of __ksize() to mm/slab.h mm/slub: limit number of node partial slabs only in cache creation mm/slub: refactor deactivate_slab()

 include/linux/slab.h |  36 ++++++------
 mm/slab.c            |  51 ++++++++---------
 mm/slab.h            |  21 +++++++
 mm/slab_common.c     |  20 +++++++
 mm/slob.c            |   1 -
 mm/slub.c            | 130 ++++++++++++-------------------------------
 6 files changed, 114 insertions(+), 145 deletions(-)

--
2.33.1

#16 lore raw
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

Quoted 7 lines
Quoted 3 lines

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Oh sorry, I didn't understand why 0 was safer when I was reading it.

Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?

Quoted 25 lines
Quoted 25 lines

Hello, these are cleanup patches for slab.
Please consider them for slab-next :)

Any comments will be appreciated.
Thanks.

Hyeonggon Yoo (5):
mm/slab: kmalloc: pass requests larger than order-1 page to page
allocator
mm/sl[au]b: unify __ksize()
mm/sl[auo]b: move definition of __ksize() to mm/slab.h mm/slub: limit number of node partial slabs only in cache creation mm/slub: refactor deactivate_slab()

 include/linux/slab.h |  36 ++++++------
 mm/slab.c            |  51 ++++++++---------
 mm/slab.h            |  21 +++++++
 mm/slab_common.c     |  20 +++++++
 mm/slob.c            |   1 -
 mm/slub.c            | 130 ++++++++++++-------------------------------
 6 files changed, 114 insertions(+), 145 deletions(-)

--
2.33.1

Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)
#17 lore raw
Quoted 15 lines · Hyeonggon Yoo
Quoted 13 lines · Marco Elver
Quoted 11 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Quoted 8 lines
Quoted 3 lines

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Oh sorry, I didn't understand why 0 was safer when I was reading it.

Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?

Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0. The intuition here is that the caller has a pointer to an invalid object, and wants to use ksize() to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by having ksize() return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj); touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.

The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)

#18 lore raw
Quoted 22 lines · Marco Elver
Quoted 21 lines · Hyeonggon Yoo
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

Quoted 7 lines
Quoted 3 lines

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Oh sorry, I didn't understand why 0 was safer when I was reading it.

Quoted 7 lines
Quoted 2 lines

Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?

Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.

Hmm, but how paranoid do we have to be? Patch 1 converts SLAB to use kmalloc_large(). So it's now legitimate to have objects allocated by SLAB's kmalloc() that don't have a slab folio flag set, and their size is folio_size(). It would be more common than getting a bogus pointer, so should we return 0 just because a bogus pointer is possible? If we do that, then KASAN will fail to unpoison legitimate kmalloc_large() objects, no? What I suggested earlier is we could make the checks more precise - if folio_size() is smaller or equal order-1 page, then it's bogus because we only do kmalloc_large() for >order-1. If the object pointer is not to the beginning of the folio, then it's bogus, because kmalloc_large() returns the beginning of the folio. Then in these case we return 0, but otherwise we should return folio_size()?

Quoted 12 lines

The intuition here is that the caller has a pointer to an invalid object, and wants to use ksize() to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by having ksize() return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj); touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.

The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)

#19 lore raw
Quoted 32 lines · Vlastimil Babka
Quoted 22 lines · Marco Elver
Quoted 21 lines · Hyeonggon Yoo
Quoted 9 lines · Marco Elver
Quoted 8 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

Quoted 7 lines
Quoted 3 lines

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Oh sorry, I didn't understand why 0 was safer when I was reading it.

Quoted 7 lines
Quoted 2 lines

Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?

Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.

Quoted 5 lines

Hmm, but how paranoid do we have to be? Patch 1 converts SLAB to use kmalloc_large(). So it's now legitimate to have objects allocated by SLAB's kmalloc() that don't have a slab folio flag set, and their size is folio_size(). It would be more common than getting a bogus pointer, so should we return 0 just because a bogus pointer is possible?

No of course not, which is why I asked in the earlier email if it's a "definitive failure case".

Quoted 22 lines

If we do that,
then KASAN will fail to unpoison legitimate kmalloc_large() objects, no? What I suggested earlier is we could make the checks more precise - if folio_size() is smaller or equal order-1 page, then it's bogus because we only do kmalloc_large() for >order-1. If the object pointer is not to the beginning of the folio, then it's bogus, because kmalloc_large() returns the beginning of the folio. Then in these case we return 0, but otherwise we should return folio_size()?

Quoted 12 lines

The intuition here is that the caller has a pointer to an invalid object, and wants to use ksize() to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by having ksize() return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj); touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.

The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)

#20 lore raw
In reply to Marco Elver
Quoted 30 lines · Marco Elver
Quoted 15 lines · Hyeonggon Yoo
Quoted 13 lines · Marco Elver
Quoted 11 lines · Hyeonggon Yoo

Changes from v1:
Now SLAB passes requests larger than order-1 page to page allocator.

Adjusted comments from Matthew, Vlastimil, Rientjes. Thank you for feedback!

BTW, I have no idea what __ksize() should return when an object that is not allocated from slab is passed. both 0 and folio_size() seems wrong to me.

Quoted 8 lines
Quoted 3 lines

Didn't we say 0 would be the safer of the two options?
https://lkml.kernel.org/r/[email protected]

Oh sorry, I didn't understand why 0 was safer when I was reading it.

Reading again, 0 is safer because kasan does not unpoison for wrongly passed object, right?

Not quite. KASAN can tell if something is wrong, i.e. invalid object. Similarly, if you are able to tell if the passed pointer is not a valid object some other way, you can do something better - namely, return 0.

Quoted 8 lines

The intuition here is that the caller has a pointer to an invalid object, and wants to use ksize() to determine its size, and most likely access all those bytes. Arguably, at that point the kernel is already in a degrading state. But we can try to not let things get worse by having ksize() return 0, in the hopes that it will stop corrupting more memory. It won't work in all cases, but should avoid things like "s = ksize(obj); touch_all_bytes(obj, s)" where the size bounds the memory accessed corrupting random memory.

Oh, it's to prevent to corrupt memory further in failure case, like memset(obj, 0, s);

Quoted 3 lines

The other reason is that a caller could actually check the size, and if 0, do something else. Few callers will do so, because nobody expects that their code has a bug. :-)

and making it able to check errors by caller.
Thank you so much for kind explanation.

I'll add what Vlastimil suggested in next series. Thanks!

Signature · 2 lines
Thank you, You are awesome!
Hyeonggon :-)