slub: move kmem_cache_node into it's own cacheline

This patch is meant to improve the performance of SLUB by moving the local kmem_cache_node lock into it's own cacheline separate from kmem_cache. This is accomplished by simply removing the local_node when NUMA is enabled.

On my system with 2 nodes I saw around a 5% performance increase w/ hackbench times dropping from 6.2 seconds to 5.9 seconds on average. I suspect the performance gain would increase as the number of nodes increases, but I do not have the data to currently back that up.

---

 include/linux/slub_def.h |    9 +++------
 mm/slub.c                |   33 +++++++++++----------------------
 2 files changed, 14 insertions(+), 28 deletions(-)

index 0249d41..7d7bf5a 100644
@@ -75,12 +75,6 @@ struct kmem_cache {
 	int offset;		/* Free pointer offset. */
 	struct kmem_cache_order_objects oo;
 
-	/*
-	 * Avoid an extra cache line for UP, SMP and for the node local to
-	 * struct kmem_cache.
-	 */
-	struct kmem_cache_node local_node;
-
 	/* Allocation and freeing of slabs */
 	struct kmem_cache_order_objects max;
 	struct kmem_cache_order_objects min;

@@ -102,6 +96,9 @@ struct kmem_cache {
 	 */
 	int remote_node_defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];
+#else
+	/* Avoid an extra cache line for UP */
+	struct kmem_cache_node local_node;
 #endif
 };
 

index 461314b..8af03de 100644
@@ -2141,7 +2141,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
 	for_each_node_state(node, N_NORMAL_MEMORY) {
 		struct kmem_cache_node *n = s->node[node];
-		if (n && n != &s->local_node)
+		if (n)
 			kmem_cache_free(kmalloc_caches, n);
 		s->node[node] = NULL;
 	}

@@ -2150,33 +2150,22 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 {
 	int node;
-	int local_node;
-
-	if (slab_state >= UP && (s < kmalloc_caches ||
-			s >= kmalloc_caches + KMALLOC_CACHES))
-		local_node = page_to_nid(virt_to_page(s));
-	else
-		local_node = 0;
 
 	for_each_node_state(node, N_NORMAL_MEMORY) {
 		struct kmem_cache_node *n;
 
-		if (local_node == node)
-			n = &s->local_node;
-		else {
-			if (slab_state == DOWN) {
-				early_kmem_cache_node_alloc(gfpflags, node);
-				continue;
-			}
-			n = kmem_cache_alloc_node(kmalloc_caches,
-							gfpflags, node);
-
-			if (!n) {
-				free_kmem_cache_nodes(s);
-				return 0;
-			}
+		if (slab_state == DOWN) {
+			early_kmem_cache_node_alloc(gfpflags, node);
+			continue;
+		}
+		n = kmem_cache_alloc_node(kmalloc_caches,
+						gfpflags, node);
 
+		if (!n) {
+			free_kmem_cache_nodes(s);
+			return 0;
 		}
+
 		s->node[node] = n;
 		init_kmem_cache_node(n, s);
 	}

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Thanks for the fix, Alexander!

Yanmin and Alex, can I have your Tested-by or Acked-by please so we can close "[Bug #15713] hackbench regression due to commit 9dfc6e68bfe6e" after this patch is merged?

Pekka

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

c3VyZS4gZ2xhZCB0byBhIGZpeCBvbiB0aGlzISANCiANCg0KLS0tLS1PcmlnaW5hbCBNZXNzYWdl LS0tLS0NCkZyb206IFBla2thIEVuYmVyZyBbbWFpbHRvOnBlbmJlcmdAY3MuaGVsc2lua2kuZmld IA0KU2VudDogMjAxMMTqNdTCMjLI1SAxNjozNg0KVG86IER1eWNrLCBBbGV4YW5kZXIgSA0KQ2M6 IGNsQGxpbnV4LmNvbTsgbGludXgtbW1Aa3ZhY2sub3JnOyBTaGksIEFsZXg7IHlhbm1pbl96aGFu Z0BsaW51eC5pbnRlbC5jb207IGFrcG1AbGludXgtZm91bmRhdGlvbi5vcmc7IGxpbnV4LWtlcm5l bEB2Z2VyLmtlcm5lbC5vcmc7IHJqd0BzaXNrLnBsDQpTdWJqZWN0OiBSZTogW1BBVENIIHYyXSBz bHViOiBtb3ZlIGttZW1fY2FjaGVfbm9kZSBpbnRvIGl0J3Mgb3duIGNhY2hlbGluZQ0KDQpBbGV4 YW5kZXIgRHV5Y2sgd3JvdGU6DQo+IFRoaXMgcGF0Y2ggaXMgbWVhbnQgdG8gaW1wcm92ZSB0aGUg cGVyZm9ybWFuY2Ugb2YgU0xVQiBieSBtb3ZpbmcgdGhlIA0KPiBsb2NhbCBrbWVtX2NhY2hlX25v ZGUgbG9jayBpbnRvIGl0J3Mgb3duIGNhY2hlbGluZSBzZXBhcmF0ZSBmcm9tIGttZW1fY2FjaGUu DQo+IFRoaXMgaXMgYWNjb21wbGlzaGVkIGJ5IHNpbXBseSByZW1vdmluZyB0aGUgbG9jYWxfbm9k ZSB3aGVuIE5VTUEgaXMgZW5hYmxlZC4NCj4gDQo+IE9uIG15IHN5c3RlbSB3aXRoIDIgbm9kZXMg SSBzYXcgYXJvdW5kIGEgNSUgcGVyZm9ybWFuY2UgaW5jcmVhc2Ugdy8gDQo+IGhhY2tiZW5jaCB0 aW1lcyBkcm9wcGluZyBmcm9tIDYuMiBzZWNvbmRzIHRvIDUuOSBzZWNvbmRzIG9uIGF2ZXJhZ2Uu ICANCj4gSSBzdXNwZWN0IHRoZSBwZXJmb3JtYW5jZSBnYWluIHdvdWxkIGluY3JlYXNlIGFzIHRo ZSBudW1iZXIgb2Ygbm9kZXMgDQo+IGluY3JlYXNlcywgYnV0IEkgZG8gbm90IGhhdmUgdGhlIGRh dGEgdG8gY3VycmVudGx5IGJhY2sgdGhhdCB1cC4NCj4gDQo+IFNpZ25lZC1vZmYtYnk6IEFsZXhh bmRlciBEdXljayA8YWxleGFuZGVyLmguZHV5Y2tAaW50ZWwuY29tPg0KDQpUaGFua3MgZm9yIHRo ZSBmaXgsIEFsZXhhbmRlciENCg0KWWFubWluIGFuZCBBbGV4LCBjYW4gSSBoYXZlIHlvdXIgVGVz dGVkLWJ5IG9yIEFja2VkLWJ5IHBsZWFzZSBzbyB3ZSBjYW4gY2xvc2UgIltCdWcgIzE1NzEzXSBo YWNrYmVuY2ggcmVncmVzc2lvbiBkdWUgdG8gY29tbWl0IDlkZmM2ZTY4YmZlNmUiIA0KYWZ0ZXIg dGhpcyBwYXRjaCBpcyBtZXJnZWQ/DQoNCgkJCVBla2thDQo=

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>