Skip to contentSkip to conversation

mm/slub: deduplicate NUMA policy calculation in allocation paths

PATCH Superseded Hao Li 2 participants 4 messages Jun 18 – Jun 19

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Hao Li

author reviewer
#0 lore raw

PATCH 1/1 mm/slub: deduplicate NUMA policy calculation in allocation paths

Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant computations.

Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.

---
 mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)
mm/slub.c +29 -43
index 62e9cd46916f..45e9f379b7da 100644
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
+static __always_inline int apply_numa_policy(int node)
+{
+#ifdef CONFIG_NUMA
+	if (static_branch_unlikely(&strict_numa) &&
+			node == NUMA_NO_NODE) {
+
+		struct mempolicy *mpol = current->mempolicy;
+
+		if (mpol) {
+			/*
+			 * Special BIND rule support. If the local node
+			 * is in permitted set then do not redirect
+			 * to a particular node.
+			 * Otherwise we apply the memory policy to get
+			 * the node we need to allocate on.
+			 */
+			if (mpol->mode != MPOL_BIND ||
+					!node_isset(numa_mem_id(), mpol->nodes))
+				node = mempolicy_slab_node();
+		}
+	}
+#endif
+	return node;
+}
+
 static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
 			       const struct slab_alloc_context *ac)
 {
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	object = ___slab_alloc(s, gfpflags, node, ac);
 
 	return object;
@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
 	bool node_requested;
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			 node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
 
 	/*
@@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
 	if (unlikely(object))
 		goto out;
 
+	node = apply_numa_policy(node);
+
 	object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
 
 	if (unlikely(!object))
@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
 		 */
 		return NULL;
 
+	node = apply_numa_policy(node);
+
 	ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
 	if (ret)
 		goto success;
Signature · 1 line
2.54.0
author reviewer
#1 lore raw
Quoted 65 lines · Hao Li

Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant computations.

Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.

---
 mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)
mm/slub.c +25 -14
index 62e9cd46916f..45e9f379b7da 100644
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
+static __always_inline int apply_numa_policy(int node)
+{
+#ifdef CONFIG_NUMA
+	if (static_branch_unlikely(&strict_numa) &&
+			node == NUMA_NO_NODE) {
+
+		struct mempolicy *mpol = current->mempolicy;
+
+		if (mpol) {
+			/*
+			 * Special BIND rule support. If the local node
+			 * is in permitted set then do not redirect
+			 * to a particular node.
+			 * Otherwise we apply the memory policy to get
+			 * the node we need to allocate on.
+			 */
+			if (mpol->mode != MPOL_BIND ||
+					!node_isset(numa_mem_id(), mpol->nodes))
+				node = mempolicy_slab_node();
+		}
+	}
+#endif
+	return node;
+}
+
 static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
 			       const struct slab_alloc_context *ac)
 {
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */

mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4558

Quoted 10 lines
mm/slub.c +0 -7
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	object = ___slab_alloc(s, gfpflags, node, ac);
 
 	return object;

oh!, __slab_alloc_node is almost an empty wrapper
Maybe I need to eliminate it completely...

Quoted 50 lines
mm/slub.c +4 -22
@@ -4756,28 +4760,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
 	bool node_requested;
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			 node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
 
 	/*
@@ -4927,6 +4909,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
 	if (unlikely(object))
 		goto out;
 
+	node = apply_numa_policy(node);
+
 	object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
 
 	if (unlikely(!object))
@@ -5430,6 +5414,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
 		 */
 		return NULL;
 
+	node = apply_numa_policy(node);
+
 	ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
 	if (ret)
 		goto success;
Signature · 1 line
2.54.0
maintainer
#2 lore raw
In reply to Hao Li
Quoted 4 lines · Hao Li

Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant computations.

It uses a static key, so probably just slightly larger code when disabled.

By inlining both __slab_alloc_node and alloc_from_pcs(), I assume the compiler would have deduplicated it, but there is a patch in slab/for-next that makes it not inlined anymore :)

Quoted 2 lines

Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.

Nice! I think there's no reason why we shouldn't do this.

Quoted 10 lines
---
 mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)
mm/slub.c +0 -0
index 62e9cd46916f..45e9f379b7da 100644

mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4526

Quoted 4 lines
mm/slub.c +1 -0
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
+static __always_inline int apply_numa_policy(int node)

apply_numa_policy() is bit confusing because we usually don't apply mempolicy for each object (unless strict_numa is set), but rather when grabbing new slabs.

perhaps apply_strict_numa[_policy]() will be a better name?

Signature · 2 lines
Cheers,
Harry / Hyeonggon
author reviewer
#3 lore raw
Quoted 9 lines · Harry Yoo
Quoted 4 lines · Hao Li

Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant computations.

It uses a static key, so probably just slightly larger code when disabled.

Yeah, make sense. The performance impact in this case should be negligible.

Quoted 4 lines

By inlining both __slab_alloc_node and alloc_from_pcs(), I assume the compiler would have deduplicated it, but there is a patch in slab/for-next that makes it not inlined anymore :)

Thanks, I hadn't thought about it from that angle before, and that's a interesting point.

Building on that, I thought about it a bit more. While in theory the compiler might be able to help us here, some of the information inside mempolicy_slab_node() seems only could be determined at runtime. So I suspect the compiler might not be able to infer that the two code blocks are equivalent and deduplicate it.

Quoted 5 lines
Quoted 3 lines

Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead.

Nice! I think there's no reason why we shouldn't do this.

Thanks!

Quoted 14 lines
Quoted 14 lines
---
 mm/slub.c | 72 ++++++++++++++++++++++---------------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)
mm/slub.c +0 -0
index 62e9cd46916f..45e9f379b7da 100644
@@ -4523,32 +4523,36 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
Quoted 5 lines

mm/slub.c static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, · line 4526

Quoted 1 line
mm/slub.c +1 -0
+static __always_inline int apply_numa_policy(int node)

apply_numa_policy() is bit confusing because we usually don't apply mempolicy for each object (unless strict_numa is set), but rather when grabbing new slabs.

Yes, this make sense. it's only for strict numa mode.

Quoted 2 lines

perhaps apply_strict_numa[_policy]() will be a better name?

Agree, apply_strict_numa_policy is indeed a better name!

Signature · 2 lines
Thanks,
Hao