slub: move kmem_cache_node into it's own cacheline

This patch is meant to improve the performance of SLUB by moving the local kmem_cache_node lock into it's own cacheline separate from kmem_cache. This is accomplished by simply removing the local_node when NUMA is enabled.

On my system with 2 nodes I saw around a 5% performance increase w/ hackbench times dropping from 6.2 seconds to 5.9 seconds on average.  I suspect the performance gain would increase as the number of nodes increases, but I do not have the data to currently back that up.

Signed-off-by Alexander Duyck <[email protected]>

---

 include/linux/slub_def.h |   11 ++++-------
 mm/slub.c                |   33 +++++++++++----------------------

 2 files changed, 15 insertions(+), 29 deletions(-)

include/linux/slub_def.h +1 -1

index 0249d41..e6217bb 100644
@@ -52,7 +52,7 @@ struct kmem_cache_node {
        atomic_long_t total_objects;
        struct list_head full;
  #endif
-};
+} ____cacheline_internodealigned_in_smp;
 
  /*
  * Word size structure that can be atomically updated or read and that

@@ -75,12 +75,6 @@ struct kmem_cache {

       int offset;             /* Free pointer offset. */ struct kmem_cache_order_objects oo;

-       /*
-        * Avoid an extra cache line for UP, SMP and for the node local to
-        * struct kmem_cache.
-        */
-       struct kmem_cache_node local_node;
-
       /* Allocation and freeing of slabs */
       struct kmem_cache_order_objects max;
       struct kmem_cache_order_objects min;

include/linux/slub_def.h +3 -0

@@ -102,6 +96,9 @@ struct kmem_cache {
         */
        int remote_node_defrag_ratio;
        struct kmem_cache_node *node[MAX_NUMNODES];
+#else
+       /* Avoid an extra cache line for UP */
+       struct kmem_cache_node local_node;
  #endif
  };
 

mm/slub.c +11 -23

index 461314b..8af03de 100644
@@ -2141,7 +2141,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = s->node[node];
-               if (n && n != &s->local_node)
+               if (n)
                        kmem_cache_free(kmalloc_caches, n);
                s->node[node] = NULL;
        }

@@ -2150,33 +2150,22 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
  static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
  {
        int node;
-       int local_node;
-
-       if (slab_state >= UP && (s < kmalloc_caches ||
-                       s >= kmalloc_caches + KMALLOC_CACHES))
-               local_node = page_to_nid(virt_to_page(s));
-       else
-               local_node = 0;
 
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;
 
-               if (local_node == node)
-                       n = &s->local_node;
-               else {
-                       if (slab_state == DOWN) {
-                               early_kmem_cache_node_alloc(gfpflags, node);
-                               continue;
-                       }
-                       n = kmem_cache_alloc_node(kmalloc_caches,
-                                                       gfpflags, node);
-
-                       if (!n) {
-                               free_kmem_cache_nodes(s);
-                               return 0;
-                       }
+               if (slab_state == DOWN) {
+                       early_kmem_cache_node_alloc(gfpflags, node);
+                       continue;
+               }
+               n = kmem_cache_alloc_node(kmalloc_caches,
+                                               gfpflags, node);
 
+               if (!n) {
+                       free_kmem_cache_nodes(s);
+                       return 0;
                }
+
                s->node[node] = n;
                init_kmem_cache_node(n, s);
        }
 
--

To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Pekka Enberg
Sent: Friday, May 21, 2010 1:00 PM
To: Duyck, Alexander H
Cc: [email protected]; [email protected]; Shi, Alex; Zhang Yanmin
Subject: Re: [PATCH] slub: move kmem_cache_node into it's own cacheline

Quoted 10 lines · Alexander Duyck

This patch is meant to improve the performance of SLUB by moving the local kmem_cache_node lock into it's own cacheline separate from kmem_cache. This is accomplished by simply removing the local_node when NUMA is enabled.

On my system with 2 nodes I saw around a 5% performance increase w/ hackbench times dropping from 6.2 seconds to 5.9 seconds on average.  I suspect the performance gain would increase as the number of nodes increases, but I do not have the data to currently back that up.

Signed-off-by Alexander Duyck <[email protected]>

Yanmin, does this fix the hackbench regression for you?

Quoted 100 lines

---

 include/linux/slub_def.h |   11 ++++-------
 mm/slub.c                |   33 +++++++++++----------------------

 2 files changed, 15 insertions(+), 29 deletions(-)

include/linux/slub_def.h +1 -1

index 0249d41..e6217bb 100644
@@ -52,7 +52,7 @@ struct kmem_cache_node {
        atomic_long_t total_objects;
        struct list_head full;
  #endif
-};
+} ____cacheline_internodealigned_in_smp;
 
  /*
  * Word size structure that can be atomically updated or read and that

@@ -75,12 +75,6 @@ struct kmem_cache {

       int offset;             /* Free pointer offset. */ struct kmem_cache_order_objects oo;

-       /*
-        * Avoid an extra cache line for UP, SMP and for the node local to
-        * struct kmem_cache.
-        */
-       struct kmem_cache_node local_node;
-
       /* Allocation and freeing of slabs */
       struct kmem_cache_order_objects max;
       struct kmem_cache_order_objects min;

include/linux/slub_def.h +3 -0

@@ -102,6 +96,9 @@ struct kmem_cache {
         */
        int remote_node_defrag_ratio;
        struct kmem_cache_node *node[MAX_NUMNODES];
+#else
+       /* Avoid an extra cache line for UP */
+       struct kmem_cache_node local_node;
  #endif
  };
 

mm/slub.c +11 -22

index 461314b..8af03de 100644
@@ -2141,7 +2141,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n = s->node[node];
-               if (n && n != &s->local_node)
+               if (n)
                        kmem_cache_free(kmalloc_caches, n);
                s->node[node] = NULL;
        }

@@ -2150,33 +2150,22 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
  static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
  {
        int node;
-       int local_node;
-
-       if (slab_state >= UP && (s < kmalloc_caches ||
-                       s >= kmalloc_caches + KMALLOC_CACHES))
-               local_node = page_to_nid(virt_to_page(s));
-       else
-               local_node = 0;
 
        for_each_node_state(node, N_NORMAL_MEMORY) {
                struct kmem_cache_node *n;
 
-               if (local_node == node)
-                       n = &s->local_node;
-               else {
-                       if (slab_state == DOWN) {
-                               early_kmem_cache_node_alloc(gfpflags, node);
-                               continue;
-                       }
-                       n = kmem_cache_alloc_node(kmalloc_caches,
-                                                       gfpflags, node);
-
-                       if (!n) {
-                               free_kmem_cache_nodes(s);
-                               return 0;
-                       }
+               if (slab_state == DOWN) {
+                       early_kmem_cache_node_alloc(gfpflags, node);
+                       continue;
+               }
+               n = kmem_cache_alloc_node(kmalloc_caches,
+                                               gfpflags, node);
 
+               if (!n) {
+                       free_kmem_cache_nodes(s);
+                       return 0;
                }
+
                s->node[node] = n;
                init_kmem_cache_node(n, s);
        }

Quoted 6 lines

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

-----Original Message-----
From: Pekka Enberg [mailto:[email protected]]
Sent: Tuesday, May 25, 2010 2:14 AM
To: Shi, Alex
Cc: Duyck, Alexander H; [email protected]; [email protected]; Zhang Yanmin; Chen, Tim C
Subject: Re: [PATCH] slub: move kmem_cache_node into it's own cacheline

Applied, thanks!

include/linux/slub_def.h struct kmem_cache_node { · line 55

Quoted 9 lines · Alexander Duyck

include/linux/slub_def.h +2 -1

index 0249d41..e6217bb 100644 --- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h

@@ -52,7 +52,7 @@ struct kmem_cache_node {
 	atomic_long_t total_objects;
 	struct list_head full;
 #endif
-};
+} ____cacheline_internodealigned_in_smp;

What does this do? Leftovers?

Quoted 13 lines · Christoph Lameter

include/linux/slub_def.h struct kmem_cache_node { · line 55

Quoted 9 lines · Alexander Duyck

include/linux/slub_def.h +2 -1

index 0249d41..e6217bb 100644 --- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h

@@ -52,7 +52,7 @@ struct kmem_cache_node {
 	atomic_long_t total_objects;
 	struct list_head full;
 #endif
-};
+} ____cacheline_internodealigned_in_smp;

What does this do? Leftovers?

It aligns it to the correct size so that no two instances can occupy a shared cacheline. I put that in place to avoid any false sharing of the objects should they fit into a shared cacheline on a NUMA system.

struct kmem_cache is allocated without any alignment so the alignment spec does not work.

If you want this then you also need to align struct kmem_cache. internode aligned would require the kmem_cache to be page aligned. So lets drop the hunk from this patch for now. A separate patch may convince us to merge aligning kmem_cache_node within kmem_cache.

Quoted 7 lines · Christoph Lameter

struct kmem_cache is allocated without any alignment so the alignment spec does not work.

If you want this then you also need to align struct kmem_cache. internode aligned would require the kmem_cache to be page aligned. So lets drop the hunk from this patch for now. A separate patch may convince us to merge aligning kmem_cache_node within kmem_cache.

I will pull that hunk out, test it, and resubmit within the next hour or so if everything looks good.