mm: slub: call account_slab_page() after slab page initialization
Subsystems (3)
PATCH 1/2 mm: slub: call account_slab_page() after slab page initialization
It's convenient to have page->objects initialized before calling into account_slab_page(). In particular, this information can be used to pre-alloc the obj_cgroup vector.
Let's call account_slab_page() a bit later, after the initialization of page->objects.
This commit doesn't bring any functional change, but is required for further optimizations.
--- mm/slub.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
mm/slub.c +2 -3
@@ -1622,9 +1622,6 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,else page =__alloc_pages_node (node, flags, order); -if (page) -account_slab_page (page, order, s); -return page; }
@@ -1777,6 +1774,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) page->objects =oo_objects (oo); +account_slab_page (page,oo_order (oo), s, flags); + page->slab_cache = s;__SetPageSlab (page);if (page_is_pfmemalloc (page))
Signature · 1 line
2.26.2
PATCH 2/2 mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.
But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
--- include/linux/memcontrol.h | 14 ++++++++++---- mm/memcontrol.c | 4 ++-- mm/slab.c | 2 +- mm/slab.h | 14 ++++++++++---- 4 files changed, 23 insertions(+), 11 deletions(-)
include/linux/memcontrol.h +10 -4
@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)* set_page_objcgs - associate a page with a object cgroups vector * @page: a pointer to the page struct * @objcgs: a pointer to the object cgroups vector +* @atomic: save the value atomically * * Atomically associates a page with a vector of object cgroups. */ static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) +struct obj_cgroup **objcgs,bool atomic) { -return !cmpxchg (&page->memcg_data,0 , (unsigned long )objcgs | -MEMCG_DATA_OBJCGS ); +unsigned long memcg_data = (unsigned long ) objcgs |MEMCG_DATA_OBJCGS ; + +if (atomic) +return !cmpxchg (&page->memcg_data,0 , memcg_data); + + page->memcg_data = memcg_data; +return true; }#else static inline struct obj_cgroup **page_objcgs (struct page *page)
@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) }static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) +struct obj_cgroup **objcgs,bool atomic) {return true; }
mm/memcontrol.c +2 -2
@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp) +gfp_t gfp,bool atomic) {unsigned int objects =objs_per_slab_page (s, page);void *vec;
@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,if (!vec)return -ENOMEM ; -if (!set_page_objcgs (page, vec)) +if (!set_page_objcgs (page, vec, atomic))kfree (vec);else kmemleak_not_leak (vec);
mm/slab.c +1 -1
@@ -1380,7 +1380,7 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,return NULL ; } -account_slab_page (page, cachep->gfporder, cachep); +account_slab_page (page, cachep->gfporder, cachep, flags);__SetPageSlab (page);/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ if (sk_memalloc_socks () &&page_is_pfmemalloc (page))
mm/slab.h +10 -4
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp); +gfp_t gfp,bool atomic);static inline void memcg_free_page_obj_cgroups (struct page *page) {
@@ -307,7 +307,8 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, page =virt_to_head_page (p[i]);if (!page_objcgs (page) && -memcg_alloc_page_obj_cgroups (page, s, flags)) { +memcg_alloc_page_obj_cgroups (page, s, flags, + true)) {obj_cgroup_uncharge (objcg,obj_full_size (s));continue ; }
@@ -371,7 +372,8 @@ static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) }static inline int memcg_alloc_page_obj_cgroups (struct page *page, -struct kmem_cache *s,gfp_t gfp) +struct kmem_cache *s,gfp_t gfp, +bool atomic) {return 0 ; }
@@ -412,8 +414,12 @@ static inline struct kmem_cache *virt_to_cache(const void *obj) } static __always_inline void account_slab_page(struct page *page, int order, -struct kmem_cache *s) +struct kmem_cache *s, +gfp_t gfp) { +if (memcg_kmem_enabled () && (s->flags &SLAB_ACCOUNT )) +memcg_alloc_page_obj_cgroups (page, s, gfp, false); +mod_node_page_state (page_pgdat (page),cache_vmstat_idx (s),PAGE_SIZE << order); }
Signature · 1 line
2.26.2
Andrew Morton
maintainerRe: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 8 lines · On Tue, 10 Nov 2020 11:57:53 -0800 Roman Gushchin <[email protected]> wrote:
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.
Quoted 8 lines
But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
Was there any measurable performance change from this?
Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 15 lines · Andrew Morton
Quoted 13 lines · On Tue, 10 Nov 2020 11:57:53 -0800 Roman Gushchin <[email protected]> wrote:
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.
Quoted 5 lines
Quoted 3 lines
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
Was there any measurable performance change from this?
A very simplistic benchmark (allocating 10000000 64-bytes objects in a row) shows ~15% win. In the real life it seems that most workloads are not very sensitive to the speed of (accounted) slab allocations.
Johannes Weiner
maintainerRe: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
In reply to Roman GushchinQuoted 13 lines · Roman Gushchin
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.
Quoted 5 lines
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
Signed-off-by Roman Gushchin <[email protected]>
That's a nice optimization!
Some comments inline:
include/linux/memcontrol.h static inline struct obj_cgroup **page_objcgs_check(struct page *page) · line 493
Quoted 11 lines
include/linux/memcontrol.h +2
-1@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page)* set_page_objcgs - associate a page with a object cgroups vector * @page: a pointer to the page struct * @objcgs: a pointer to the object cgroups vector +* @atomic: save the value atomically * * Atomically associates a page with a vector of object cgroups. */ static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) +struct obj_cgroup **objcgs,bool atomic)
bool parameters make callsites pretty hard to understand - unless the function interface obviously has two modes (read vs write etc.), which isn't the case here.
Quoted 35 lines
include/linux/memcontrol.h +8
-3-return !cmpxchg (&page->memcg_data,0 , (unsigned long )objcgs | - MEMCG_DATA_OBJCGS); +unsigned long memcg_data = (unsigned long ) objcgs |MEMCG_DATA_OBJCGS ; + +if (atomic) +return !cmpxchg (&page->memcg_data,0 , memcg_data); + + page->memcg_data = memcg_data; + return true; }#else static inline struct obj_cgroup **page_objcgs (struct page *page)@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) }static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) +struct obj_cgroup **objcgs,bool atomic) {return true; }mm/memcontrol.c +1
-1@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp) +gfp_t gfp,bool atomic) {unsigned int objects =objs_per_slab_page (s, page);void *vec;
mm/memcontrol.c int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, · line 2890
Quoted 8 lines
mm/memcontrol.c +1
-1@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,if (!vec)return -ENOMEM ; -if (!set_page_objcgs (page, vec)) +if (!set_page_objcgs (page, vec, atomic))kfree (vec);else kmemleak_not_leak (vec);
The life of page->memcg_data and this optimization could use a central comment somewhere, because it's hard to understand what's going on from the code alone. This function here seems like a good place?
I don't see a way to eliminate the bool on the allocation function, but maybe it could be more descriptive. Maybe bool slab_account?
set_page_objcgs() can be inlined at this point. It made some sense to abstract away the atomics with setter and matching getter, but with a non-atomic mode, inlining makes things clearer and allows for better in-place documentation in the sole callsite.
How about something like this?
vec = kcalloc(...);
memcg_data = (unsigned long)vec | MEMCG_DATA_OBJCGS;
/*
* Set up the objcg vector for the page.
*
* When only some objects in a slab are tracked (think GFP_ACCOUNT
* kmalloc allocations), the objcg vector is set up when the first
* tracked object in the slab page is allocated. Multiple concurrent
* slab allocations can race to this, so synchronization is required.
*
* When SLAB_ACCOUNT is set on the cache, however, all objects in the
* slab page will be tracked, and the vector is allocated along with
* the page itself, while it's still exclusive; no atomics necessary.
*/
if (slab_account) {
page->memcg_data = memcg_data;
} else {
if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
/* Somebody else beat us, use their vec */
kfree(vec);
return 0;
}
}
kmemleak_not_leak(vec);
return 0;Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 88 lines · Johannes Weiner
Quoted 13 lines · Roman Gushchin
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.Quoted 5 lines
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
Signed-off-by Roman Gushchin <[email protected]>That's a nice optimization!
Some comments inline:
Review static inline struct obj_cgroup **page_objcgs_check(struct page *page) · line 493
Quoted 11 lines
(quoted diff) +2
-1@@ -485,14 +485,20 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) * set_page_objcgs - associate a page with a object cgroups vector * @page: a pointer to the page struct * @objcgs: a pointer to the object cgroups vector + * @atomic: save the value atomically * * Atomically associates a page with a vector of object cgroups. */ static inline bool set_page_objcgs(struct page *page, - struct obj_cgroup **objcgs) + struct obj_cgroup **objcgs, bool atomic)bool parameters make callsites pretty hard to understand - unless the function interface obviously has two modes (read vs write etc.), which isn't the case here.
Quoted 35 lines
(quoted diff) +8
-3- return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs | - MEMCG_DATA_OBJCGS); + unsigned long memcg_data = (unsigned long) objcgs | MEMCG_DATA_OBJCGS; + + if (atomic) + return !cmpxchg(&page->memcg_data, 0, memcg_data); + + page->memcg_data = memcg_data; + return true; } #else static inline struct obj_cgroup **page_objcgs(struct page *page)@@ -506,7 +512,7 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) } static inline bool set_page_objcgs(struct page *page, - struct obj_cgroup **objcgs) + struct obj_cgroup **objcgs, bool atomic) { return true; }mm/memcontrol.c +1
-1@@ -2874,7 +2874,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp) +gfp_t gfp,bool atomic) {unsigned int objects =objs_per_slab_page (s, page);void *vec;mm/memcontrol.c int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, · line 2890
Quoted 8 lines
mm/memcontrol.c +1
-1@@ -2884,7 +2884,7 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,if (!vec)return -ENOMEM ; -if (!set_page_objcgs (page, vec)) +if (!set_page_objcgs (page, vec, atomic))kfree (vec);else kmemleak_not_leak (vec);The life of
page->memcg_dataand this optimization could use a central comment somewhere, because it's hard to understand what's going on from the code alone. This function here seems like a good place?
Quoted 7 lines
I don't see a way to eliminate the bool on the allocation function, but maybe it could be more descriptive. Maybe bool
slab_account?
set_page_objcgs()can be inlined at this point. It made some sense to abstract away the atomics with setter and matching getter, but with a non-atomic mode, inlining makes things clearer and allows for better in-place documentation in the sole callsite.
I agree, inlining makes sense here.
I'd prefer not to spill details about SLAB_ACCOUNT into memcg_alloc_page_obj_cgroups(). What's important there is only whether we have an exclusive access to the page or not.
How about the version below?
Thanks!
--
From 8b28d91475d54c552e503e66f169e1e00475c856 Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab
caches with SLAB_ACCOUNT
In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.
But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc.
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
v2: inline set_page_objcgs() and add some comments, by Johannes
--- include/linux/memcontrol.h | 19 ------------------- mm/memcontrol.c | 23 +++++++++++++++++++---- mm/slab.c | 2 +- mm/slab.h | 14 ++++++++++---- 4 files changed, 30 insertions(+), 28 deletions(-)
include/linux/memcontrol.h +0 -19
@@ -481,19 +481,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); } -/* -* set_page_objcgs - associate a page with a object cgroups vector -* @page: a pointer to the page struct -* @objcgs: a pointer to the object cgroups vector -* -* Atomically associates a page with a vector of object cgroups. -*/ -static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) -{ -return !cmpxchg (&page->memcg_data,0 , (unsigned long )objcgs | -MEMCG_DATA_OBJCGS ); -}#else static inline struct obj_cgroup **page_objcgs (struct page *page) {
@@ -504,12 +491,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) {return NULL ; } - -static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs) -{ -return true; -}#endif static __always_inline bool memcg_stat_item_in_bytes(int idx)
mm/memcontrol.c +19 -4
@@ -2874,9 +2874,10 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp) +gfp_t gfp,bool new_page) {unsigned int objects =objs_per_slab_page (s, page); +unsigned long memcg_data;void *vec; vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
@@ -2884,11 +2885,25 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,if (!vec)return -ENOMEM ; -if (!set_page_objcgs (page, vec)) + memcg_data = (unsigned long ) vec |MEMCG_DATA_OBJCGS ; +if (new_page) { +/* +* If the slab page is brand new and nobody can yet access +* it's memcg_data, no synchronization is required and +* memcg_data can be simply assigned. +*/ + page->memcg_data = memcg_data; + }else if (cmpxchg (&page->memcg_data,0 , memcg_data)) { +/* +* If the slab page is already in use, somebody can allocate +* and assign obj_cgroups in parallel. In this case the existing +* objcg vector should be reused. +*/ kfree (vec); -else -kmemleak_not_leak (vec); +return 0 ; + } +kmemleak_not_leak (vec);return 0 ; }
mm/slab.c +1 -1
@@ -1380,7 +1380,7 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,return NULL ; } -account_slab_page (page, cachep->gfporder, cachep); +account_slab_page (page, cachep->gfporder, cachep, flags);__SetPageSlab (page);/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ if (sk_memalloc_socks () &&page_is_pfmemalloc (page))
mm/slab.h +10 -4
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp); +gfp_t gfp,bool new_page);static inline void memcg_free_page_obj_cgroups (struct page *page) {
@@ -307,7 +307,8 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, page =virt_to_head_page (p[i]);if (!page_objcgs (page) && -memcg_alloc_page_obj_cgroups (page, s, flags)) { +memcg_alloc_page_obj_cgroups (page, s, flags, + false)) {obj_cgroup_uncharge (objcg,obj_full_size (s));continue ; }
@@ -371,7 +372,8 @@ static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) }static inline int memcg_alloc_page_obj_cgroups (struct page *page, -struct kmem_cache *s,gfp_t gfp) +struct kmem_cache *s,gfp_t gfp, +bool new_page) {return 0 ; }
@@ -412,8 +414,12 @@ static inline struct kmem_cache *virt_to_cache(const void *obj) } static __always_inline void account_slab_page(struct page *page, int order, -struct kmem_cache *s) +struct kmem_cache *s, +gfp_t gfp) { +if (memcg_kmem_enabled () && (s->flags &SLAB_ACCOUNT )) +memcg_alloc_page_obj_cgroups (page, s, gfp, true); +mod_node_page_state (page_pgdat (page),cache_vmstat_idx (s),PAGE_SIZE << order); }
Signature · 1 line
2.26.2
Andrew Morton
maintainerRe: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 19 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line
From
8b28d91475d54c552e503e66f169e1e00475c856Mon Sep 17 00:00:00 2001From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocateobj_cgroupsfor slab
caches withSLAB_ACCOUNTIn general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.
Quoted 5 lines
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
v2: inline
set_page_objcgs()and add some comments, by Johannes
Had me confused! I was looking for the inlined function set_page_objcgs(). I think "open-code" is a better term here than "inline".
Here's the -fix:
From: Roman Gushchin <[email protected]>
Subject: mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2
open-code set_page_objcgs() and add some comments, by Johannes
---
include/linux/memcontrol.h | 25 ------------------------- mm/memcontrol.c | 23 +++++++++++++++++++---- mm/slab.h | 8 ++++---- 3 files changed, 23 insertions(+), 33 deletions(-)
include/linux/memcontrol.h renamed from include/linux/memcontrol.h~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +0 -25
@@ -480,25 +480,6 @@ static inline struct obj_cgroup **page_o return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); } -/* -* set_page_objcgs - associate a page with a object cgroups vector -* @page: a pointer to the page struct -* @objcgs: a pointer to the object cgroups vector -* @atomic: save the value atomically -* -* Atomically associates a page with a vector of object cgroups. -*/ -static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs,bool atomic) -{ -unsigned long memcg_data = (unsigned long ) objcgs |MEMCG_DATA_OBJCGS ; - -if (atomic) -return !cmpxchg (&page->memcg_data,0 , memcg_data); - - page->memcg_data = memcg_data; -return true; -}#else static inline struct obj_cgroup **page_objcgs (struct page *page) {
@@ -509,12 +490,6 @@ static inline struct obj_cgroup **page_o {return NULL ; } - -static inline bool set_page_objcgs (struct page *page, -struct obj_cgroup **objcgs,bool atomic) -{ -return true; -}#endif static __always_inline bool memcg_stat_item_in_bytes(int idx)
mm/memcontrol.c renamed from mm/memcontrol.c~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +19 -4
@@ -2879,9 +2879,10 @@ static void commit_charge(struct page *p#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp,bool atomic) +gfp_t gfp,bool new_page) {unsigned int objects =objs_per_slab_page (s, page); +unsigned long memcg_data;void *vec; vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
@@ -2889,11 +2890,25 @@ int memcg_alloc_page_obj_cgroups(structif (!vec)return -ENOMEM ; -if (!set_page_objcgs (page, vec, atomic)) + memcg_data = (unsigned long ) vec |MEMCG_DATA_OBJCGS ; +if (new_page) { +/* +* If the slab page is brand new and nobody can yet access +* it's memcg_data, no synchronization is required and +* memcg_data can be simply assigned. +*/ + page->memcg_data = memcg_data; + }else if (cmpxchg (&page->memcg_data,0 , memcg_data)) { +/* +* If the slab page is already in use, somebody can allocate +* and assign obj_cgroups in parallel. In this case the existing +* objcg vector should be reused. +*/ kfree (vec); -else -kmemleak_not_leak (vec); +return 0 ; + } +kmemleak_not_leak (vec);return 0 ; }
mm/slab.h renamed from mm/slab.h~mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2 +4 -4
@@ -240,7 +240,7 @@ static inline bool kmem_cache_debug_flag#ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s, -gfp_t gfp,bool atomic); +gfp_t gfp,bool new_page);static inline void memcg_free_page_obj_cgroups (struct page *page) {
@@ -308,7 +308,7 @@ static inline void memcg_slab_post_allocif (!page_objcgs (page) &&memcg_alloc_page_obj_cgroups (page, s, flags, - true)) { + false)) {obj_cgroup_uncharge (objcg,obj_full_size (s));continue ; }
@@ -373,7 +373,7 @@ static inline struct mem_cgroup *memcg_fstatic inline int memcg_alloc_page_obj_cgroups (struct page *page,struct kmem_cache *s,gfp_t gfp, -bool atomic) +bool new_page) {return 0 ; }
@@ -418,7 +418,7 @@ static __always_inline void account_slab gfp_t gfp) {if (memcg_kmem_enabled () && (s->flags &SLAB_ACCOUNT )) -memcg_alloc_page_obj_cgroups (page, s, gfp, false); +memcg_alloc_page_obj_cgroups (page, s, gfp, true);mod_node_page_state (page_pgdat (page),cache_vmstat_idx (s),PAGE_SIZE << order);
_
Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 25 lines · Andrew Morton
Quoted 23 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line
From
8b28d91475d54c552e503e66f169e1e00475c856Mon Sep 17 00:00:00 2001From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocateobj_cgroupsfor slab
caches withSLAB_ACCOUNTIn general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
Quoted 5 lines
Quoted 1 line
v2: inline
set_page_objcgs()and add some comments, by JohannesHad me confused! I was looking for the inlined function
set_page_objcgs(). I think "open-code" is a better term here than "inline".
Sorry for the confusion! And thanks for picking up the new version!
Roman
Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
In reply to Andrew MortonQuoted 38 lines · Andrew Morton
Quoted 19 lines · On Thu, 12 Nov 2020 16:19:26 -0800 Roman Gushchin <[email protected]> wrote:
Quoted 1 line
From
8b28d91475d54c552e503e66f169e1e00475c856Mon Sep 17 00:00:00 2001From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocateobj_cgroupsfor slab
caches withSLAB_ACCOUNTIn general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.Quoted 5 lines
In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
v2: inline
set_page_objcgs()and add some comments, by JohannesHad me confused! I was looking for the inlined function
set_page_objcgs(). I think "open-code" is a better term here than "inline".Here's the -fix:
From: Roman Gushchin <[email protected]>
Subject: mm-memcg-slab-pre-allocate-obj_cgroups-for-slab-caches-with-slab_account-v2open-code
set_page_objcgs()and add some comments, by Johannes
Quoted 7 lines
Signed-off-by Roman Gushchin <[email protected]>Cc Christoph Lameter <[email protected]>Cc Johannes Weiner <[email protected]>Cc Michal Hocko <[email protected]>Cc Shakeel Butt <[email protected]>Signed-off-by Andrew Morton <[email protected]>
Thanks! For the combined patch:
Shakeel Butt
Reviewed-byRe: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
In reply to Roman GushchinQuoted 1 line · Roman Gushchin
[snip]
Quoted 27 lines
From
8b28d91475d54c552e503e66f169e1e00475c856Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocateobj_cgroupsfor slab
caches withSLAB_ACCOUNTIn general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
v2: inline
set_page_objcgs()and add some comments, by JohannesSigned-off-by Roman Gushchin <[email protected]>
[snip]
mm/slab.h static inline struct kmem_cache *virt_to_cache(const void *obj) · line 422
Quoted 9 lines
static
__always_inlinevoid account_slab_page(struct page *page, int order,mm/slab.h +5
-1- struct kmem_cache *s) + struct kmem_cache *s, + gfp_t gfp) { + if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT)) + memcg_alloc_page_obj_cgroups(page, s, gfp, true); +
I was wondering why not add (gfp & __GFP_ACCOUNT) check as well but it seems like for that some additional plumbing is required.
Anyways:
Re: [PATCH 2/2] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT
Quoted 41 lines · Shakeel Butt
Quoted 1 line · Roman Gushchin
[snip]
Quoted 27 lines
From
8b28d91475d54c552e503e66f169e1e00475c856Mon Sep 17 00:00:00 2001
From: Roman Gushchin <[email protected]>
Date: Wed, 16 Sep 2020 15:43:48 -0700
Subject: [PATCH v2 2/2] mm: memcg/slab: pre-allocateobj_cgroupsfor slab
caches withSLAB_ACCOUNTIn general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an
obj_cgroupvector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensivecmpxchg()to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu.But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a
SLAB_ACCOUNTflag set. It includes such popular objects likevm_area_struct,anon_vma,task_struct, etc.In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else.
v2: inline
set_page_objcgs()and add some comments, by JohannesSigned-off-by Roman Gushchin <[email protected]>[snip]
mm/slab.h static inline struct kmem_cache *virt_to_cache(const void *obj) · line 422
Quoted 9 lines
static
__always_inlinevoid account_slab_page(struct page *page, int order,mm/slab.h +5
-1- struct kmem_cache *s) + struct kmem_cache *s, + gfp_t gfp) { + if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT)) + memcg_alloc_page_obj_cgroups(page, s, gfp, true); +
Quoted 2 lines
I was wondering why not add (gfp &
__GFP_ACCOUNT) check as well but it seems like for that some additional plumbing is required.
Yes, it's doable, but requires passing another argument through the whole stack.
Quoted 4 lines
Anyways:
Reviewed-by Shakeel Butt <[email protected]>
Thank you!
Quoted 11 lines · Roman Gushchin
It's convenient to have
page->objectsinitialized before calling intoaccount_slab_page(). In particular, this information can be used to pre-alloc theobj_cgroupvector.Let's call
account_slab_page()a bit later, after the initialization ofpage->objects.This commit doesn't bring any functional change, but is required for further optimizations.
Signed-off-by Roman Gushchin <[email protected]>
Quoted 5 lines · Roman Gushchin
It's convenient to have
page->objectsinitialized before calling intoaccount_slab_page(). In particular, this information can be used to pre-alloc theobj_cgroupvector.
Quoted 7 lines
Let's call
account_slab_page()a bit later, after the initialization ofpage->objects.This commit doesn't bring any functional change, but is required for further optimizations.
Signed-off-by Roman Gushchin <[email protected]>