Skip to contentSkip to conversation

mm: slub: call account_slab_page() after slab page initialization

PATCH In review Roman Gushchin 4 participants 13 messages Nov 10, 2020 – Nov 17, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 mm: slub: call account_slab_page() after slab page initialization

It's convenient to have page->objects initialized before calling into account_slab_page(). In particular, this information can be used to pre-alloc the obj_cgroup vector.

Let's call account_slab_page() a bit later, after the initialization of page->objects.

This commit doesn't bring any functional change, but is required for further optimizations.

---
 mm/slub.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
mm/slub.c +2 -3
index 8c9fa24ebbf3..ccdbb62e025d 100644
@@ -1622,9 +1622,6 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
 	else
 		page = __alloc_pages_node(node, flags, order);
 
-	if (page)
-		account_slab_page(page, order, s);
-
 	return page;
 }
 
@@ -1777,6 +1774,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
 	page->objects = oo_objects(oo);
 
+	account_slab_page(page, oo_order(oo), s, flags);
+
 	page->slab_cache = s;
 	__SetPageSlab(page);
 	if (page_is_pfmemalloc(page))
Signature · 1 line
2.26.2
#1 lore raw

PATCH 2/2 mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

---
 include/linux/memcontrol.h | 14 ++++++++++----
 mm/memcontrol.c            |  4 ++--
 mm/slab.c                  |  2 +-
 mm/slab.h                  | 14 ++++++++++----
 4 files changed, 23 insertions(+), 11 deletions(-)
include/linux/memcontrol.h +10 -4
index 20108e426f84..8271f11152e6 100644
@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
  * set_page_objcgs - associate a page with a object cgroups vector
  * @page: a pointer to the page struct
  * @objcgs: a pointer to the object cgroups vector
+ * @atomic: save the value atomically
  *
  * Atomically associates a page with a vector of object cgroups.
  */
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)
 {
-	return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs |
-			MEMCG_DATA_OBJCGS);
+	unsigned long memcg_data = (unsigned long) objcgs | MEMCG_DATA_OBJCGS;
+
+	if (atomic)
+		return !cmpxchg(&page->memcg_data, 0, memcg_data);
+
+	page->memcg_data = memcg_data;
+	return true;
 }
 #else
 static inline struct obj_cgroup **page_objcgs(struct page *page)
@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
 }
 
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)
 {
 	return true;
 }
mm/memcontrol.c +2 -2
index 69a2893a6455..37bffd336235 100644
@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp)
+				 gfp_t gfp, bool atomic)
 {
 	unsigned int objects = objs_per_slab_page(s, page);
 	void *vec;
@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
 	if (!vec)
 		return -ENOMEM;
 
-	if (!set_page_objcgs(page, vec))
+	if (!set_page_objcgs(page, vec, atomic))
 		kfree(vec);
 	else
 		kmemleak_not_leak(vec);
mm/slab.c +1 -1
index c0ea4b1c7088..df0299e1d0b9 100644
@@ -1380,7 +1380,7 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
 		return NULL;
 	}
 
-	account_slab_page(page, cachep->gfporder, cachep);
+	account_slab_page(page, cachep->gfporder, cachep, flags);
 	__SetPageSlab(page);
 	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
 	if (sk_memalloc_socks() && page_is_pfmemalloc(page))
mm/slab.h +10 -4
index c73050654b8a..f1d6ba09b630 100644
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp);
+				 gfp_t gfp, bool atomic);
 
 static inline void memcg_free_page_obj_cgroups(struct page *page)
 {
@@ -307,7 +307,8 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
 			page = virt_to_head_page(p[i]);
 
 			if (!page_objcgs(page) &&
-			    memcg_alloc_page_obj_cgroups(page, s, flags)) {
+			    memcg_alloc_page_obj_cgroups(page, s, flags,
+							 true)) {
 				obj_cgroup_uncharge(objcg, obj_full_size(s));
 				continue;
 			}
@@ -371,7 +372,8 @@ static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
 }
 
 static inline int memcg_alloc_page_obj_cgroups(struct page *page,
-					       struct kmem_cache *s, gfp_t gfp)
+					       struct kmem_cache *s, gfp_t gfp,
+					       bool atomic)
 {
 	return 0;
 }
@@ -412,8 +414,12 @@ static inline struct kmem_cache *virt_to_cache(const void *obj)
 }
 
 static __always_inline void account_slab_page(struct page *page, int order,
-					      struct kmem_cache *s)
+					      struct kmem_cache *s,
+					      gfp_t gfp)
 {
+	if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
+		memcg_alloc_page_obj_cgroups(page, s, gfp, false);
+
 	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
 			    PAGE_SIZE << order);
 }
Signature · 1 line
2.26.2
maintainer
#2 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 8 lines · On Tue, 10 Nov 2020 11:57:53 -0800 Roman Gushchin <[email protected]> wrote:

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

Quoted 8 lines

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

Was there any measurable performance change from this?

#3 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 15 lines · Andrew Morton
Quoted 13 lines · On Tue, 10 Nov 2020 11:57:53 -0800 Roman Gushchin <[email protected]> wrote:

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

Quoted 5 lines
Quoted 3 lines

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

Was there any measurable performance change from this?

A very simplistic benchmark (allocating 10000000 64-bytes objects in a row) shows ~15% win. In the real life it seems that most workloads are not very sensitive to the speed of (accounted) slab allocations.

maintainer
#4 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

In reply to Roman Gushchin
Quoted 13 lines · Roman Gushchin

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

Quoted 5 lines

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

That's a nice optimization!

Some comments inline:

include/linux/memcontrol.h static inline struct obj_cgroup **page_objcgs_check(struct page *page) · line 493

Quoted 11 lines
include/linux/memcontrol.h +2 -1
@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
  * set_page_objcgs - associate a page with a object cgroups vector
  * @page: a pointer to the page struct
  * @objcgs: a pointer to the object cgroups vector
+ * @atomic: save the value atomically
  *
  * Atomically associates a page with a vector of object cgroups.
  */
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)

bool parameters make callsites pretty hard to understand - unless the function interface obviously has two modes (read vs write etc.), which isn't the case here.

Quoted 35 lines
include/linux/memcontrol.h +8 -3
-	return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs |
-			MEMCG_DATA_OBJCGS);
+	unsigned long memcg_data = (unsigned long) objcgs | MEMCG_DATA_OBJCGS;
+
+	if (atomic)
+		return !cmpxchg(&page->memcg_data, 0, memcg_data);
+
+	page->memcg_data = memcg_data;
+	return true;
 }
 #else
 static inline struct obj_cgroup **page_objcgs(struct page *page)
@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
 }
 
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)
 {
 	return true;
 }
mm/memcontrol.c +1 -1
index 69a2893a6455..37bffd336235 100644
@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp)
+				 gfp_t gfp, bool atomic)
 {
 	unsigned int objects = objs_per_slab_page(s, page);
 	void *vec;

mm/memcontrol.c int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, · line 2890

Quoted 8 lines
mm/memcontrol.c +1 -1
@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
 	if (!vec)
 		return -ENOMEM;
 
-	if (!set_page_objcgs(page, vec))
+	if (!set_page_objcgs(page, vec, atomic))
 		kfree(vec);
 	else
 		kmemleak_not_leak(vec);

The life of page->memcg_data and this optimization could use a central comment somewhere, because it's hard to understand what's going on from the code alone. This function here seems like a good place?

I don't see a way to eliminate the bool on the allocation function, but maybe it could be more descriptive. Maybe bool slab_account?

set_page_objcgs() can be inlined at this point. It made some sense to abstract away the atomics with setter and matching getter, but with a non-atomic mode, inlining makes things clearer and allows for better in-place documentation in the sole callsite.

How about something like this?

	vec = kcalloc(...);

	memcg_data = (unsigned long)vec | MEMCG_DATA_OBJCGS;
	/*
	 * Set up the objcg vector for the page.
	 *
	 * When only some objects in a slab are tracked (think GFP_ACCOUNT
	 * kmalloc allocations), the objcg vector is set up when the first
	 * tracked object in the slab page is allocated. Multiple concurrent
	 * slab allocations can race to this, so synchronization is required.
	 *
	 * When SLAB_ACCOUNT is set on the cache, however, all objects in the
	 * slab page will be tracked, and the vector is allocated along with
	 * the page itself, while it's still exclusive; no atomics necessary.
	 */
	if (slab_account) {
		page->memcg_data = memcg_data;
	} else {
		if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
			/* Somebody else beat us, use their vec */
			kfree(vec);
			return 0;
		}
	}
	kmemleak_not_leak(vec);
	return 0;
#5 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 88 lines · Johannes Weiner
Quoted 13 lines · Roman Gushchin

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

Quoted 5 lines

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

That's a nice optimization!

Some comments inline:

Review static inline struct obj_cgroup **page_objcgs_check(struct page *page) · line 493

Quoted 11 lines
(quoted diff) +2 -1
@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
  * set_page_objcgs - associate a page with a object cgroups vector
  * @page: a pointer to the page struct
  * @objcgs: a pointer to the object cgroups vector
+ * @atomic: save the value atomically
  *
  * Atomically associates a page with a vector of object cgroups.
  */
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)

bool parameters make callsites pretty hard to understand - unless the function interface obviously has two modes (read vs write etc.), which isn't the case here.

Quoted 35 lines
(quoted diff) +8 -3
-	return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs |
-			MEMCG_DATA_OBJCGS);
+	unsigned long memcg_data = (unsigned long) objcgs | MEMCG_DATA_OBJCGS;
+
+	if (atomic)
+		return !cmpxchg(&page->memcg_data, 0, memcg_data);
+
+	page->memcg_data = memcg_data;
+	return true;
 }
 #else
 static inline struct obj_cgroup **page_objcgs(struct page *page)
@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
 }
 
 static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
+				   struct obj_cgroup **objcgs, bool atomic)
 {
 	return true;
 }
mm/memcontrol.c +1 -1
index 69a2893a6455..37bffd336235 100644
@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp)
+				 gfp_t gfp, bool atomic)
 {
 	unsigned int objects = objs_per_slab_page(s, page);
 	void *vec;

mm/memcontrol.c int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, · line 2890

Quoted 8 lines
mm/memcontrol.c +1 -1
@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
 	if (!vec)
 		return -ENOMEM;
 
-	if (!set_page_objcgs(page, vec))
+	if (!set_page_objcgs(page, vec, atomic))
 		kfree(vec);
 	else
 		kmemleak_not_leak(vec);

The life of page->memcg_data and this optimization could use a central comment somewhere, because it's hard to understand what's going on from the code alone. This function here seems like a good place?

Quoted 7 lines

I don't see a way to eliminate the bool on the allocation function, but maybe it could be more descriptive. Maybe bool slab_account?

set_page_objcgs() can be inlined at this point. It made some sense to abstract away the atomics with setter and matching getter, but with a non-atomic mode, inlining makes things clearer and allows for better in-place documentation in the sole callsite.

I agree, inlining makes sense here.

I'd prefer not to spill details about SLAB_ACCOUNT into memcg_alloc_page_obj_cgroups(). What's important there is only whether we have an exclusive access to the page or not.

How about the version below?

Thanks!

--

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

v2: inline set_page_objcgs() and add some comments, by Johannes

---
 include/linux/memcontrol.h | 19 -------------------
 mm/memcontrol.c            | 23 +++++++++++++++++++----
 mm/slab.c                  |  2 +-
 mm/slab.h                  | 14 ++++++++++----
 4 files changed, 30 insertions(+), 28 deletions(-)
include/linux/memcontrol.h +0 -19
index 20108e426f84..bfcdcb2d2976 100644
@@ -481,19 +481,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
 	return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
 }
 
-/*
- * set_page_objcgs - associate a page with a object cgroups vector
- * @page: a pointer to the page struct
- * @objcgs: a pointer to the object cgroups vector
- *
- * Atomically associates a page with a vector of object cgroups.
- */
-static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
-{
-	return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs |
-			MEMCG_DATA_OBJCGS);
-}
 #else
 static inline struct obj_cgroup **page_objcgs(struct page *page)
 {
@@ -504,12 +491,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)
 {
 	return NULL;
 }
-
-static inline bool set_page_objcgs(struct page *page,
-					struct obj_cgroup **objcgs)
-{
-	return true;
-}
 #endif
 
 static __always_inline bool memcg_stat_item_in_bytes(int idx)
mm/memcontrol.c +19 -4
index 69a2893a6455..6ca8c23d7816 100644
@@ -2874,9 +2874,10 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp)
+				 gfp_t gfp, bool new_page)
 {
 	unsigned int objects = objs_per_slab_page(s, page);
+	unsigned long memcg_data;
 	void *vec;
 
 	vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
@@ -2884,11 +2885,25 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
 	if (!vec)
 		return -ENOMEM;
 
-	if (!set_page_objcgs(page, vec))
+	memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
+	if (new_page) {
+		/*
+		 * If the slab page is brand new and nobody can yet access
+		 * it's memcg_data, no synchronization is required and
+		 * memcg_data can be simply assigned.
+		 */
+		page->memcg_data = memcg_data;
+	} else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
+		/*
+		 * If the slab page is already in use, somebody can allocate
+		 * and assign obj_cgroups in parallel. In this case the existing
+		 * objcg vector should be reused.
+		 */
 		kfree(vec);
-	else
-		kmemleak_not_leak(vec);
+		return 0;
+	}
 
+	kmemleak_not_leak(vec);
 	return 0;
 }
 
mm/slab.c +1 -1
index c0ea4b1c7088..df0299e1d0b9 100644
@@ -1380,7 +1380,7 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
 		return NULL;
 	}
 
-	account_slab_page(page, cachep->gfporder, cachep);
+	account_slab_page(page, cachep->gfporder, cachep, flags);
 	__SetPageSlab(page);
 	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
 	if (sk_memalloc_socks() && page_is_pfmemalloc(page))
mm/slab.h +10 -4
index c73050654b8a..59aeb0d9f11b 100644
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp);
+				 gfp_t gfp, bool new_page);
 
 static inline void memcg_free_page_obj_cgroups(struct page *page)
 {
@@ -307,7 +307,8 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
 			page = virt_to_head_page(p[i]);
 
 			if (!page_objcgs(page) &&
-			    memcg_alloc_page_obj_cgroups(page, s, flags)) {
+			    memcg_alloc_page_obj_cgroups(page, s, flags,
+							 false)) {
 				obj_cgroup_uncharge(objcg, obj_full_size(s));
 				continue;
 			}
@@ -371,7 +372,8 @@ static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
 }
 
 static inline int memcg_alloc_page_obj_cgroups(struct page *page,
-					       struct kmem_cache *s, gfp_t gfp)
+					       struct kmem_cache *s, gfp_t gfp,
+					       bool new_page)
 {
 	return 0;
 }
@@ -412,8 +414,12 @@ static inline struct kmem_cache *virt_to_cache(const void *obj)
 }
 
 static __always_inline void account_slab_page(struct page *page, int order,
-					      struct kmem_cache *s)
+					      struct kmem_cache *s,
+					      gfp_t gfp)
 {
+	if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
+		memcg_alloc_page_obj_cgroups(page, s, gfp, true);
+
 	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
 			    PAGE_SIZE << order);
 }
Signature · 1 line
2.26.2
maintainer
#6 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 19 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001

From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

Quoted 5 lines

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

v2: inline set_page_objcgs() and add some comments, by Johannes

Had me confused! I was looking for the inlined function set_page_objcgs(). I think "open-code" is a better term here than "inline".

Here's the -fix:

From: Roman Gushchin <[email protected]>
Subject: mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2

open-code set_page_objcgs() and add some comments, by Johannes

---

 include/linux/memcontrol.h |   25 -------------------------
 mm/memcontrol.c            |   23 +++++++++++++++++++----
 mm/slab.h                  |    8 ++++----
 3 files changed, 23 insertions(+), 33 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +0 -25
@@ -480,25 +480,6 @@ static inline struct obj_cgroup **page_o
 	return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
 }
 
-/*
- * set_page_objcgs - associate a page with a object cgroups vector
- * @page: a pointer to the page struct
- * @objcgs: a pointer to the object cgroups vector
- * @atomic: save the value atomically
- *
- * Atomically associates a page with a vector of object cgroups.
- */
-static inline bool set_page_objcgs(struct page *page,
-				   struct obj_cgroup **objcgs, bool atomic)
-{
-	unsigned long memcg_data = (unsigned long) objcgs | MEMCG_DATA_OBJCGS;
-
-	if (atomic)
-		return !cmpxchg(&page->memcg_data, 0, memcg_data);
-
-	page->memcg_data = memcg_data;
-	return true;
-}
 #else
 static inline struct obj_cgroup **page_objcgs(struct page *page)
 {
@@ -509,12 +490,6 @@ static inline struct obj_cgroup **page_o
 {
 	return NULL;
 }
-
-static inline bool set_page_objcgs(struct page *page,
-				   struct obj_cgroup **objcgs, bool atomic)
-{
-	return true;
-}
 #endif
 
 static __always_inline bool memcg_stat_item_in_bytes(int idx)
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +19 -4
@@ -2879,9 +2879,10 @@ static void commit_charge(struct page *p
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp, bool atomic)
+				 gfp_t gfp, bool new_page)
 {
 	unsigned int objects = objs_per_slab_page(s, page);
+	unsigned long memcg_data;
 	void *vec;
 
 	vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
@@ -2889,11 +2890,25 @@ int memcg_alloc_page_obj_cgroups(struct
 	if (!vec)
 		return -ENOMEM;
 
-	if (!set_page_objcgs(page, vec, atomic))
+	memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
+	if (new_page) {
+		/*
+		 * If the slab page is brand new and nobody can yet access
+		 * it's memcg_data, no synchronization is required and
+		 * memcg_data can be simply assigned.
+		 */
+		page->memcg_data = memcg_data;
+	} else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
+		/*
+		 * If the slab page is already in use, somebody can allocate
+		 * and assign obj_cgroups in parallel. In this case the existing
+		 * objcg vector should be reused.
+		 */
 		kfree(vec);
-	else
-		kmemleak_not_leak(vec);
+		return 0;
+	}
 
+	kmemleak_not_leak(vec);
 	return 0;
 }
 
mm/slab.h renamed from mm/slab.h~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +4 -4
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flag
 
 #ifdef CONFIG_MEMCG_KMEM
 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
-				 gfp_t gfp, bool atomic);
+				 gfp_t gfp, bool new_page);
 
 static inline void memcg_free_page_obj_cgroups(struct page *page)
 {
@@ -308,7 +308,7 @@ static inline void memcg_slab_post_alloc
 
 			if (!page_objcgs(page) &&
 			    memcg_alloc_page_obj_cgroups(page, s, flags,
-							 true)) {
+							 false)) {
 				obj_cgroup_uncharge(objcg, obj_full_size(s));
 				continue;
 			}
@@ -373,7 +373,7 @@ static inline struct mem_cgroup *memcg_f
 
 static inline int memcg_alloc_page_obj_cgroups(struct page *page,
 					       struct kmem_cache *s, gfp_t gfp,
-					       bool atomic)
+					       bool new_page)
 {
 	return 0;
 }
@@ -418,7 +418,7 @@ static __always_inline void account_slab
 					      gfp_t gfp)
 {
 	if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
-		memcg_alloc_page_obj_cgroups(page, s, gfp, false);
+		memcg_alloc_page_obj_cgroups(page, s, gfp, true);
 
 	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
 			    PAGE_SIZE << order);

_

#7 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 25 lines · Andrew Morton
Quoted 23 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001

From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

Quoted 5 lines
Quoted 1 line

v2: inline set_page_objcgs() and add some comments, by Johannes

Had me confused! I was looking for the inlined function set_page_objcgs(). I think "open-code" is a better term here than "inline".

Sorry for the confusion! And thanks for picking up the new version!

Roman

maintainer Acked-by
#8 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

In reply to Andrew Morton
Quoted 38 lines · Andrew Morton
Quoted 19 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001

From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

Quoted 5 lines

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

v2: inline set_page_objcgs() and add some comments, by Johannes

Had me confused! I was looking for the inlined function set_page_objcgs(). I think "open-code" is a better term here than "inline".

Here's the -fix:

From: Roman Gushchin <[email protected]>
Subject: mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2

open-code set_page_objcgs() and add some comments, by Johannes

Quoted 7 lines

Thanks! For the combined patch:

Reviewed-by
#9 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

In reply to Roman Gushchin
Quoted 1 line · Roman Gushchin

[snip]

Quoted 27 lines

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

v2: inline set_page_objcgs() and add some comments, by Johannes

[snip]

mm/slab.h static inline struct kmem_cache *virt_to_cache(const void *obj) · line 422

Quoted 9 lines

static __always_inline void account_slab_page(struct page *page, int order,

mm/slab.h +5 -1
-                                             struct kmem_cache *s)
+                                             struct kmem_cache *s,
+                                             gfp_t gfp)
 {
+       if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
+               memcg_alloc_page_obj_cgroups(page, s, gfp, true);
+

I was wondering why not add (gfp & __GFP_ACCOUNT) check as well but it seems like for that some additional plumbing is required.

Anyways:

#10 lore raw

Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT

Quoted 41 lines · Shakeel Butt
Quoted 1 line · Roman Gushchin

[snip]

Quoted 27 lines

From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT

In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.

But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.

In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.

v2: inline set_page_objcgs() and add some comments, by Johannes

[snip]

mm/slab.h static inline struct kmem_cache *virt_to_cache(const void *obj) · line 422

Quoted 9 lines

static __always_inline void account_slab_page(struct page *page, int order,

mm/slab.h +5 -1
-                                             struct kmem_cache *s)
+                                             struct kmem_cache *s,
+                                             gfp_t gfp)
 {
+       if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
+               memcg_alloc_page_obj_cgroups(page, s, gfp, true);
+
Quoted 2 lines

I was wondering why not add (gfp & __GFP_ACCOUNT) check as well but it seems like for that some additional plumbing is required.

Yes, it's doable, but requires passing another argument through the whole stack.

Quoted 4 lines

Anyways:

Thank you!

maintainer Acked-by on v1 1/2
#11 lore raw
In reply to Roman Gushchin
Quoted 11 lines · Roman Gushchin

It's convenient to have page->objects initialized before calling into account_slab_page(). In particular, this information can be used to pre-alloc the obj_cgroup vector.

Let's call account_slab_page() a bit later, after the initialization of page->objects.

This commit doesn't bring any functional change, but is required for further optimizations.

Reviewed-by on v1 1/2
#12 lore raw
In reply to Roman Gushchin
Quoted 5 lines · Roman Gushchin

It's convenient to have page->objects initialized before calling into account_slab_page(). In particular, this information can be used to pre-alloc the obj_cgroup vector.

Quoted 7 lines

Let's call account_slab_page() a bit later, after the initialization of page->objects.

This commit doesn't bring any functional change, but is required for further optimizations.