Skip to contentSkip to conversation

mm: slub: init_on_free=1 should wipe freelist ptr for bulk allocations

PATCH New [email protected] 1 participant 2 messages Oct 7, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 mm: slub: init_on_free=1 should wipe freelist ptr for bulk allocations

slab_alloc_node() already zeroed out the freelist pointer if init_on_free was on.
Thibaut Sautereau noticed that the same needs to be done for kmem_cache_alloc_bulk(), which performs the allocations separately.

kmem_cache_alloc_bulk() is currently used in two places in the kernel, so this change is unlikely to have a major performance impact.

SLAB doesn't require a similar change, as auto-initialization makes the allocator store the freelist pointers off-slab.

Fixes 6471384af2a6 ("mm: security: introduce init_on_alloc=1 and init_on_free=1 boot options")

To: Andrew Morton <[email protected]>
To: Christoph Lameter <[email protected]>
Cc: Laura Abbott <[email protected]>
Cc: [email protected]
Cc: [email protected]

---
v2:
- added a missing return type to maybe_wipe_obj_freeptr() (spotted by kbuild test robot <[email protected]>)

---
 mm/slub.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)
mm/slub.c +16 -6
index 8834563cdb4b..89a69aaf58c4 100644
@@ -2669,6 +2669,17 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return p;
 }
 
+/*
+ * If the object has been wiped upon free, make sure it's fully initialized by
+ * zeroing out freelist pointer.
+ */
+static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
+						   void *obj)
+{
+	if (unlikely(slab_want_init_on_free(s)) && obj)
+		memset((void *)((char *)obj + s->offset), 0, sizeof(void *));
+}
+
 /*
  * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
  * have the fastpath folded into their functions. So no function call
@@ -2757,12 +2768,8 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s,
 		prefetch_freepointer(s, next_object);
 		stat(s, ALLOC_FASTPATH);
 	}
-	/*
-	 * If the object has been wiped upon free, make sure it's fully
-	 * initialized by zeroing out freelist pointer.
-	 */
-	if (unlikely(slab_want_init_on_free(s)) && object)
-		memset(object + s->offset, 0, sizeof(void *));
+
+	maybe_wipe_obj_freeptr(s, object);
 
 	if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object)
 		memset(object, 0, s->object_size);
@@ -3176,10 +3183,13 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 				goto error;
 
 			c = this_cpu_ptr(s->cpu_slab);
+			maybe_wipe_obj_freeptr(s, p[i]);
+
 			continue; /* goto for-loop */
 		}
 		c->freelist = get_freepointer(s, object);
 		p[i] = object;
+		maybe_wipe_obj_freeptr(s, p[i]);
 	}
 	c->tid = next_tid(c->tid);
 	local_irq_enable();
Signature · 1 line
2.23.0.581.g78d2f28ef7-goog
#1 lore raw

PATCH 2/2 lib/test_meminit: add a kmem_cache_alloc_bulk() test

Make sure allocations from kmem_cache_alloc_bulk()/kmem_cache_free_bulk() are properly initialized.

To: Andrew Morton <[email protected]>
To: Christoph Lameter <[email protected]>
Cc: [email protected]
Cc: [email protected]

---
 lib/test_meminit.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
lib/test_meminit.c +27 -0
index 9729f271d150..9742e5cb853a 100644
@@ -297,6 +297,32 @@ static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures)
 	return 1;
 }
 
+static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
+{
+	struct kmem_cache *c;
+	int i, iter, maxiter = 1024;
+	int num, bytes;
+	bool fail = false;
+	void *objects[10];
+
+	c = kmem_cache_create("test_cache", size, size, 0, NULL);
+	for (iter = 0; (iter < maxiter) && !fail; iter++) {
+		num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
+					    objects);
+		for (i = 0; i < num; i++) {
+			bytes = count_nonzero_bytes(objects[i], size);
+			if (bytes)
+				fail = true;
+			fill_with_garbage(objects[i], size);
+		}
+
+		if (num)
+			kmem_cache_free_bulk(c, num, objects);
+	}
+	*total_failures += fail;
+	return 1;
+}
+
 /*
  * Test kmem_cache allocation by creating caches of different sizes, with and
  * without constructors, with and without SLAB_TYPESAFE_BY_RCU.
@@ -318,6 +344,7 @@ static int __init test_kmemcache(int *total_failures)
 			num_tests += do_kmem_cache_size(size, ctor, rcu, zero,
 							&failures);
 		}
+		num_tests += do_kmem_cache_size_bulk(size, &failures);
 	}
 	REPORT_FAILURES_IN_FN();
 	*total_failures += failures;
Signature · 1 line
2.23.0.581.g78d2f28ef7-goog