mm/slub: deduplicate NUMA policy calculation in allocation paths
Subsystems (2)
Hao Li
reviewerPATCH 1/1 mm/slub: deduplicate NUMA policy calculation in allocation paths
Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant computations.
Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.
--- mm/slub.c | 72 ++++++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-)
mm/slub.c +29 -43
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,return object; } +static __always_inline int apply_numa_policy(int node) +{ +#ifdef CONFIG_NUMA +if (static_branch_unlikely (&strict_numa) && + node ==NUMA_NO_NODE ) { + +struct mempolicy *mpol = current->mempolicy; + +if (mpol) { +/* +* Special BIND rule support. If the local node +* is in permitted set then do not redirect +* to a particular node. +* Otherwise we apply the memory policy to get +* the node we need to allocate on. +*/ +if (mpol->mode !=MPOL_BIND || + !node_isset (numa_mem_id (), mpol->nodes)) + node =mempolicy_slab_node (); + } + } +#endif +return node; +} +static void *__slab_alloc_node (struct kmem_cache *s,gfp_t gfpflags,int node,const struct slab_alloc_context *ac) {void *object; -#ifdef CONFIG_NUMA -if (static_branch_unlikely (&strict_numa) && - node ==NUMA_NO_NODE ) { - -struct mempolicy *mpol = current->mempolicy; - -if (mpol) { -/* -* Special BIND rule support. If the local node -* is in permitted set then do not redirect -* to a particular node. -* Otherwise we apply the memory policy to get -* the node we need to allocate on. -*/ -if (mpol->mode !=MPOL_BIND || - !node_isset (numa_mem_id (), mpol->nodes)) - node =mempolicy_slab_node (); - } - } -#endif - object =___slab_alloc (s, gfpflags, node, ac);return object;
@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,bool node_requested;void *object; -#ifdef CONFIG_NUMA -if (static_branch_unlikely (&strict_numa) && - node ==NUMA_NO_NODE ) { - -struct mempolicy *mpol = current->mempolicy; - -if (mpol) { -/* -* Special BIND rule support. If the local node -* is in permitted set then do not redirect -* to a particular node. -* Otherwise we apply the memory policy to get -* the node we need to allocate on. -*/ -if (mpol->mode !=MPOL_BIND || - !node_isset (numa_mem_id (), mpol->nodes)) - - node =mempolicy_slab_node (); - } - } -#endif - node_requested =IS_ENABLED (CONFIG_NUMA ) && node !=NUMA_NO_NODE ;/*
@@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,if (unlikely (object)) goto out; + node =apply_numa_policy (node); + object =alloc_from_pcs (s, gfpflags, ac->alloc_flags, node);if (unlikely (!object))
@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f*/ return NULL ; + node =apply_numa_policy (node); + ret =alloc_from_pcs (s, gfp_flags, ac->alloc_flags, node);if (ret) goto success;
Signature · 1 line
2.54.0
Hao Li
reviewerQuoted 65 lines · Hao Li
Currently,
alloc_from_pcs()and__slab_alloc_node()both calculate the NUMA policy independently. Since they are called consecutively in paths like__kmalloc_nolock_noprof()andslab_alloc_node(), this leads to redundant computations.Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.
Signed-off-by Hao Li <[email protected]>--- mm/slub.c | 72 ++++++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-)mm/slub.c +25
-14@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,return object; } +static __always_inline int apply_numa_policy(int node) +{ +#ifdef CONFIG_NUMA +if (static_branch_unlikely (&strict_numa) && + node ==NUMA_NO_NODE ) { + +struct mempolicy *mpol = current->mempolicy; + +if (mpol) { +/* +* Special BIND rule support. If the local node +* is in permitted set then do not redirect +* to a particular node. +* Otherwise we apply the memory policy to get +* the node we need to allocate on. +*/ +if (mpol->mode !=MPOL_BIND || + !node_isset (numa_mem_id (), mpol->nodes)) + node =mempolicy_slab_node (); + } + } +#endif +return node; +} +static void *__slab_alloc_node (struct kmem_cache *s,gfp_t gfpflags,int node,const struct slab_alloc_context *ac) {void *object; -#ifdef CONFIG_NUMA -if (static_branch_unlikely (&strict_numa) && - node ==NUMA_NO_NODE ) { - -struct mempolicy *mpol = current->mempolicy; - -if (mpol) { -/* -* Special BIND rule support. If the local node -* is in permitted set then do not redirect -* to a particular node. -* Otherwise we apply the memory policy to get -* the node we need to allocate on. -*/
mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4558
Quoted 10 lines
mm/slub.c +0
-7-if (mpol->mode !=MPOL_BIND || - !node_isset (numa_mem_id (), mpol->nodes)) - node = mempolicy_slab_node(); - } - } -#endif - object =___slab_alloc (s, gfpflags, node, ac);return object;
oh!, __slab_alloc_node is almost an empty wrapper
Maybe I need to eliminate it completely...
Quoted 50 lines
mm/slub.c +4
-22@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,bool node_requested;void *object; -#ifdef CONFIG_NUMA -if (static_branch_unlikely (&strict_numa) && - node ==NUMA_NO_NODE ) { - -struct mempolicy *mpol = current->mempolicy; - -if (mpol) { -/* -* Special BIND rule support. If the local node -* is in permitted set then do not redirect -* to a particular node. -* Otherwise we apply the memory policy to get -* the node we need to allocate on. -*/ -if (mpol->mode !=MPOL_BIND || - !node_isset (numa_mem_id (), mpol->nodes)) - - node =mempolicy_slab_node (); - } - } -#endif - node_requested =IS_ENABLED (CONFIG_NUMA ) && node !=NUMA_NO_NODE ;/* @@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,if (unlikely (object)) goto out; + node =apply_numa_policy (node); + object =alloc_from_pcs (s, gfpflags, ac->alloc_flags, node);if (unlikely (!object))@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f*/ return NULL ; + node =apply_numa_policy (node); + ret =alloc_from_pcs (s, gfp_flags, ac->alloc_flags, node);if (ret) goto success;Signature · 1 line
2.54.0
Harry Yoo
maintainerQuoted 4 lines · Hao Li
Currently,
alloc_from_pcs()and__slab_alloc_node()both calculate the NUMA policy independently. Since they are called consecutively in paths like__kmalloc_nolock_noprof()andslab_alloc_node(), this leads to redundant computations.
It uses a static key, so probably just slightly larger code when disabled.
By inlining both __slab_alloc_node and alloc_from_pcs(), I assume the compiler would have deduplicated it, but there is a patch in slab/for-next that makes it not inlined anymore :)
Quoted 2 lines
Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.
Nice! I think there's no reason why we shouldn't do this.
Quoted 10 lines
Signed-off-by Hao Li <[email protected]>--- mm/slub.c | 72 ++++++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-)mm/slub.c +0
-0
mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4526
Quoted 4 lines
mm/slub.c +1
-0@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,return object; } +static __always_inline int apply_numa_policy(int node)
apply_numa_policy() is bit confusing because we usually don't apply mempolicy for each object (unless strict_numa is set), but rather when grabbing new slabs.
perhaps apply_strict_numa[_policy]() will be a better name?
Signature · 2 lines
Cheers, Harry / Hyeonggon
Hao Li
reviewerQuoted 9 lines · Harry Yoo
Quoted 4 lines · Hao Li
Currently,
alloc_from_pcs()and__slab_alloc_node()both calculate the NUMA policy independently. Since they are called consecutively in paths like__kmalloc_nolock_noprof()andslab_alloc_node(), this leads to redundant computations.It uses a static key, so probably just slightly larger code when disabled.
Yeah, make sense. The performance impact in this case should be negligible.
Quoted 4 lines
By inlining both
__slab_alloc_nodeandalloc_from_pcs(), I assume the compiler would have deduplicated it, but there is a patch in slab/for-next that makes it not inlined anymore :)
Thanks, I hadn't thought about it from that angle before, and that's a interesting point.
Building on that, I thought about it a bit more. While in theory the compiler might be able to help us here, some of the information inside mempolicy_slab_node() seems only could be determined at runtime. So I suspect the compiler might not be able to infer that the two code blocks are equivalent and deduplicate it.
Quoted 5 lines
Quoted 3 lines
Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.
Nice! I think there's no reason why we shouldn't do this.
Thanks!
Quoted 14 lines
Quoted 14 lines
Signed-off-by Hao Li <[email protected]>--- mm/slub.c | 72 ++++++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-)mm/slub.c +0
-0@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,return object; }
Quoted 5 lines
mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4526
Quoted 1 line
mm/slub.c +1
-0+static __always_inline int apply_numa_policy(int node)
apply_numa_policy()is bit confusing because we usually don't apply mempolicy for each object (unlessstrict_numais set), but rather when grabbing new slabs.
Yes, this make sense. it's only for strict numa mode.
Quoted 2 lines
perhaps apply_strict_numa[_policy]() will be a better name?
Agree, apply_strict_numa_policy is indeed a better name!
Signature · 2 lines
Thanks, Hao