Skip to contentSkip to conversation

re-shrink 'struct page' when SLUB is on.

RFC PATCH In review Dave Hansen 3 participants 9 messages Dec 11, 2013 – Dec 12, 2013

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover re-shrink 'struct page' when SLUB is on.

OK, here's the real reason I was digging around for exactly which fields in 'struct page' slub uses.

SLUB depends on a 16-byte cmpxchg for an optimization. For the purposes of this series, I'm assuming that it is a very important optimization that we desperately need to keep around.

In order to get guaranteed 16-byte alignment (required by the hardware on x86), 'struct page' is padded out from 56 to 64 bytes.

Those 8-bytes matter. We've gone to great lengths to keep 'struct page' small in the past. It's a shame that we bloat it now just for alignment reasons when we have extra space. These patches attempt _internal_ alignment instead of external alignment for slub.

I also got a bug report from some folks running a large database benchmark. Their old kernel uses slab and their new one uses slub. They were swapping and couldn't figure out why. It turned out to be the 2GB of RAM that the slub padding wastes on their system.

On my box, that 2GB cost about $200 to populate back when we bought it. I want my $200 back.

This is compile tested and lightly runtime tested. I'm curious what people think of it before we push it futher. This is on top of my previous patch to create a 'struct slab_page', but doesn't really rely on it. It could easily be done independently.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#1 lore raw

RFC PATCH 1/3 mm: slab: create helpers for slab ->freelist pointer

We have a need to move the ->freelist data around 'struct page' in order to keep a cmpxchg aligned. First step is to add an accessor function which we will hook in to in the next patch.

I'm not super-happy with how this looks. It's a bit ugly, but it does work. I'm open to some better suggestions for how to do this.

---

 linux.git-davehans/include/linux/mm_types.h |    2 
 linux.git-davehans/mm/slab.c                |   25 ++++----
 linux.git-davehans/mm/slob.c                |   34 +++++++---
 linux.git-davehans/mm/slub.c                |   87 +++++++++++++++-------------
 4 files changed, 87 insertions(+), 61 deletions(-)
include/linux/mm_types.h renamed from include/linux/mm_types.h~slub-internally-align-freelist-and-counters +1 -1
diff -puN include/linux/mm_types.h~slub-internally-align-freelist-and-counters include/linux/mm_types.h
@@ -143,7 +143,7 @@ struct slab_page {
 	void *s_mem;			/* slab first object */
 
 	/* Second double word */
-	void *freelist;		/* sl[aou]b first free object */
+	void *_freelist;		/* sl[aou]b first free object */
 
 	union {
 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
mm/slab.c renamed from mm/slab.c~slub-internally-align-freelist-and-counters +15 -10
diff -puN mm/slab.c~slub-internally-align-freelist-and-counters mm/slab.c
@@ -1950,6 +1950,16 @@ static void slab_destroy_debugcheck(stru
 }
 #endif
 
+static inline unsigned int **slab_freelist_ptr(struct slab_page *page)
+{
+	return (unsigned int **)&page->_freelist;
+}
+
+static inline unsigned int *slab_freelist(struct slab_page *page)
+{
+	return *slab_freelist_ptr(page);
+}
+
 /**
  * slab_destroy - destroy and release all objects in a slab
  * @cachep: cache pointer being destroyed
@@ -1963,7 +1973,7 @@ static void slab_destroy(struct kmem_cac
 {
 	void *freelist;
 
-	freelist = page->freelist;
+	freelist = slab_freelist(page);
 	slab_destroy_debugcheck(cachep, page);
 	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
 		struct rcu_head *head;
@@ -2549,11 +2559,6 @@ static void *alloc_slabmgmt(struct kmem_
 	return freelist;
 }
 
-static inline unsigned int *slab_freelist(struct slab_page *page)
-{
-	return (unsigned int *)(page->freelist);
-}
-
 static void cache_init_objs(struct kmem_cache *cachep,
 			    struct slab_page *page)
 {
@@ -2615,7 +2620,7 @@ static void *slab_get_obj(struct kmem_ca
 {
 	void *objp;
 
-	objp = index_to_obj(cachep, page, slab_freelist(page)[page->active]);
+	objp = index_to_obj(cachep, page, (slab_freelist(page))[page->active]);
 	page->active++;
 #if DEBUG
 	WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
@@ -2636,7 +2641,7 @@ static void slab_put_obj(struct kmem_cac
 
 	/* Verify double free bug */
 	for (i = page->active; i < cachep->num; i++) {
-		if (slab_freelist(page)[i] == objnr) {
+		if ((slab_freelist(page))[i] == objnr) {
 			printk(KERN_ERR "slab: double free detected in cache "
 					"'%s', objp %p\n", cachep->name, objp);
 			BUG();
@@ -2656,7 +2661,7 @@ static void slab_map_pages(struct kmem_c
 			   void *freelist)
 {
 	page->slab_cache = cache;
-	page->freelist = freelist;
+	*slab_freelist_ptr(page) = freelist;
 }
 
 /*
@@ -4217,7 +4222,7 @@ static void handle_slab(unsigned long *n
 
 		for (j = page->active; j < c->num; j++) {
 			/* Skip freed item */
-			if (slab_freelist(page)[j] == i) {
+			if ((slab_freelist(page))[j] == i) {
 				active = false;
 				break;
 			}
mm/slob.c renamed from mm/slob.c~slub-internally-align-freelist-and-counters +22 -12
diff -puN mm/slob.c~slub-internally-align-freelist-and-counters mm/slob.c
@@ -211,6 +211,16 @@ static void slob_free_pages(void *b, int
 	free_pages((unsigned long)b, order);
 }
 
+static inline void **slab_freelist_ptr(struct slab_page *sp)
+{
+	return &sp->_freelist;
+}
+
+static inline void *slab_freelist(struct slab_page *sp)
+{
+	return *slab_freelist_ptr(sp);
+}
+
 /*
  * Allocate a slob block within a given slob_page sp.
  */
@@ -219,7 +229,7 @@ static void *slob_page_alloc(struct slab
 	slob_t *prev, *cur, *aligned = NULL;
 	int delta = 0, units = SLOB_UNITS(size);
 
-	for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) {
+	for (prev = NULL, cur = slab_freelist(sp); ; prev = cur, cur = slob_next(cur)) {
 		slobidx_t avail = slob_units(cur);
 
 		if (align) {
@@ -243,12 +253,12 @@ static void *slob_page_alloc(struct slab
 				if (prev)
 					set_slob(prev, slob_units(prev), next);
 				else
-					sp->freelist = next;
+					*slab_freelist_ptr(sp) = next;
 			} else { /* fragment */
 				if (prev)
 					set_slob(prev, slob_units(prev), cur + units);
 				else
-					sp->freelist = cur + units;
+					*slab_freelist_ptr(sp) = cur + units;
 				set_slob(cur + units, avail - units, next);
 			}
 
@@ -322,7 +332,7 @@ static void *slob_alloc(size_t size, gfp
 
 		spin_lock_irqsave(&slob_lock, flags);
 		sp->units = SLOB_UNITS(PAGE_SIZE);
-		sp->freelist = b;
+		*slab_freelist_ptr(sp) = b;
 		INIT_LIST_HEAD(&sp->list);
 		set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
 		set_slob_page_free(sp, slob_list);
@@ -369,7 +379,7 @@ static void slob_free(void *block, int s
 	if (!slob_page_free(sp)) {
 		/* This slob page is about to become partially free. Easy! */
 		sp->units = units;
-		sp->freelist = b;
+		*slab_freelist_ptr(sp) = b;
 		set_slob(b, units,
 			(void *)((unsigned long)(b +
 					SLOB_UNITS(PAGE_SIZE)) & PAGE_MASK));
@@ -389,15 +399,15 @@ static void slob_free(void *block, int s
 	 */
 	sp->units += units;
 
-	if (b < (slob_t *)sp->freelist) {
-		if (b + units == sp->freelist) {
-			units += slob_units(sp->freelist);
-			sp->freelist = slob_next(sp->freelist);
+	if (b < (slob_t *)slab_freelist(sp)) {
+		if (b + units == slab_freelist(sp)) {
+			units += slob_units(slab_freelist(sp));
+			*slab_freelist_ptr(sp) = slob_next(slab_freelist(sp));
 		}
-		set_slob(b, units, sp->freelist);
-		sp->freelist = b;
+		set_slob(b, units, slab_freelist(sp));
+		*slab_freelist_ptr(sp) = b;
 	} else {
-		prev = sp->freelist;
+		prev = slab_freelist(sp);
 		next = slob_next(prev);
 		while (b > next) {
 			prev = next;
mm/slub.c renamed from mm/slub.c~slub-internally-align-freelist-and-counters +49 -38

266 lines not on this page · raw message

_

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#2 lore raw

Re: [RFC][PATCH 1/3] mm: slab: create helpers for slab ->freelist pointer

Quoted 8 lines · Dave Hansen

We have a need to move the ->freelist data around 'struct page' in order to keep a cmpxchg aligned. First step is to add an accessor function which we will hook in to in the next patch.

I'm not super-happy with how this looks. It's a bit ugly, but it does work. I'm open to some better suggestions for how to do this.

I think the mapping field is not used by SLUB and its ok to use since SLAB uses it for its memory pointer. Maybe you can use that to get the correct alignment? Do an and of address used for the cmpxchg with 0xffff .. ff0 to ensure proper aligment (the resulting address may overlap the mapping field).

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#3 lore raw

RFC PATCH 2/3 mm: slab: move around slab ->freelist for cmpxchg

In reply to Dave Hansen

The write-argument to cmpxchg_double() must be 16-byte aligned. We used to align 'struct page' itself in order to guarantee this, but that wastes 8-bytes per page. Instead, we take 8-bytes internal to the page before page->counters and move freelist between there and the existing 8-bytes after counters. That way, no matter how 'stuct page' itself is aligned, we can ensure that we have a 16-byte area with which to to this cmpxchg.

---

 linux.git-davehans/include/linux/mm_types.h |   17 +++++--
 linux.git-davehans/mm/slab.c                |    2 
 linux.git-davehans/mm/slab.h                |    1 
 linux.git-davehans/mm/slob.c                |    2 
 linux.git-davehans/mm/slub.c                |   67 +++++++++++++++++++++++-----
 5 files changed, 74 insertions(+), 15 deletions(-)
include/linux/mm_types.h renamed from include/linux/mm_types.h~move-around-freelist-to-align +14 -3
diff -puN include/linux/mm_types.h~move-around-freelist-to-align include/linux/mm_types.h
@@ -140,11 +140,20 @@ struct slab_page {
 	/* First double word block */
 	unsigned long flags;		/* Atomic flags, some possibly
 					 * updated asynchronously */
-	void *s_mem;			/* slab first object */
+	union {
+		void *s_mem;			/* slab first object */
+		/*
+		 * The combination of ->counters and ->freelist
+		 * need to be doubleword-aligned in order for
+		 * slub's cmpxchg_double() to work properly.
+		 * slub does not use 's_mem', so we reuse it here
+		 * so we can always have alignment no matter how
+		 * struct page is aligned.
+		 */
+		void *_freelist_first;		/* sl[aou]b first free object */
+	};
 
 	/* Second double word */
-	void *_freelist;		/* sl[aou]b first free object */
-
 	union {
 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
 	defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
@@ -173,6 +182,8 @@ struct slab_page {
 		unsigned int active;	/* SLAB */
 	};
 
+	void *_freelist_second;		/* sl[aou]b first free object */
+
 	/* Third double word block */
 	union {
 		struct {		/* slub per cpu partial pages */
mm/slab.c renamed from mm/slab.c~move-around-freelist-to-align +1 -1
diff -puN mm/slab.c~move-around-freelist-to-align mm/slab.c
@@ -1952,7 +1952,7 @@ static void slab_destroy_debugcheck(stru
 
 static inline unsigned int **slab_freelist_ptr(struct slab_page *page)
 {
-	return (unsigned int **)&page->_freelist;
+	return (unsigned int **)&page->_freelist_first;
 }
 
 static inline unsigned int *slab_freelist(struct slab_page *page)
mm/slab.h renamed from mm/slab.h~move-around-freelist-to-align +1 -0
diff -puN mm/slab.h~move-around-freelist-to-align mm/slab.h
@@ -278,3 +278,4 @@ struct kmem_cache_node {
 
 void *slab_next(struct seq_file *m, void *p, loff_t *pos);
 void slab_stop(struct seq_file *m, void *p);
+
mm/slob.c renamed from mm/slob.c~move-around-freelist-to-align +1 -1
diff -puN mm/slob.c~move-around-freelist-to-align mm/slob.c
@@ -213,7 +213,7 @@ static void slob_free_pages(void *b, int
 
 static inline void **slab_freelist_ptr(struct slab_page *sp)
 {
-	return &sp->_freelist;
+	return &sp->_freelist_first;
 }
 
 static inline void *slab_freelist(struct slab_page *sp)
mm/slub.c renamed from mm/slub.c~move-around-freelist-to-align +57 -10

92 lines not on this page · raw message

_

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#4 lore raw

Re: [RFC][PATCH 2/3] mm: slab: move around slab ->freelist for cmpxchg

Quoted 8 lines · Dave Hansen

The write-argument to cmpxchg_double() must be 16-byte aligned. We used to align 'struct page' itself in order to guarantee this, but that wastes 8-bytes per page. Instead, we take 8-bytes internal to the page before page->counters and move freelist between there and the existing 8-bytes after counters. That way, no matter how 'stuct page' itself is aligned, we can ensure that we have a 16-byte area with which to to this cmpxchg.

Well this adds additional branching to the fast paths.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#5 lore raw

Re: [RFC][PATCH 2/3] mm: slab: move around slab ->freelist for cmpxchg

Quoted 10 lines · Christoph Lameter
Quoted 7 lines · Dave Hansen

The write-argument to cmpxchg_double() must be 16-byte aligned. We used to align 'struct page' itself in order to guarantee this, but that wastes 8-bytes per page. Instead, we take 8-bytes internal to the page before page->counters and move freelist between there and the existing 8-bytes after counters. That way, no matter how 'stuct page' itself is aligned, we can ensure that we have a 16-byte area with which to to this cmpxchg.

Well this adds additional branching to the fast paths.

I don't think it *HAS* to inherently. The reason here is really that we swap the _order_ of the arguments to the cmpxchg() since their order in memory changes. Essentially, we do:

| flags | freelist  | counters |          |
| flags |           | counters | freelist |

I did this so I wouldn't have to make a helper for ->counters. But, if we also move counters around, we can do:

| flags | counters | freelist |          |
| flags |          | counters | freelist |

I believe we can do that all with plain pointer arithmetic and masks so that it won't cost any branches.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#6 lore raw

Re: [RFC][PATCH 2/3] mm: slab: move around slab ->freelist for cmpxchg

In reply to Christoph Lameter
Quoted 11 lines · Christoph Lameter
Quoted 8 lines · Dave Hansen

The write-argument to cmpxchg_double() must be 16-byte aligned. We used to align 'struct page' itself in order to guarantee this, but that wastes 8-bytes per page. Instead, we take 8-bytes internal to the page before page->counters and move freelist between there and the existing 8-bytes after counters. That way, no matter how 'stuct page' itself is aligned, we can ensure that we have a 16-byte area with which to to this cmpxchg.

Quoted 1 line

Well this adds additional branching to the fast paths.

The branch should be predictible and compare the cost of a branch (near nothing on a modern OOO CPU with low IPC code like this when predicted) to the cost of a cache miss (due to larger struct page)

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only
#7 lore raw

RFC PATCH 3/3 mm: slabs: reset page at free

In reply to Dave Hansen

We now have slub's ->freelist usage impinging on page->mapping's storage space. The buddy allocator wants ->mapping to be NULL when a page is handed back, so we have to make sure that it is cleared.

Note that slab already doeds this, so just create a common helper and have all the slabs do it this way. ->mapping is right next to ->flags, so it's virtually guaranteed to be in the L1 at this point, so this shouldn't cost very much to do in practice.

---

 linux.git-davehans/mm/slab.c |    3 +--
 linux.git-davehans/mm/slab.h |   11 +++++++++++
 linux.git-davehans/mm/slob.c |    2 +-
 linux.git-davehans/mm/slub.c |    2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)
mm/slab.c renamed from mm/slab.c~slub-reset-page-at-free +1 -2
diff -puN mm/slab.c~slub-reset-page-at-free mm/slab.c
@@ -1725,8 +1725,7 @@ static void kmem_freepages(struct kmem_c
 	BUG_ON(!PageSlab(page));
 	__ClearPageSlabPfmemalloc(page);
 	__ClearPageSlab(page);
-	page_mapcount_reset(page);
-	page->mapping = NULL;
+	slab_reset_page(page);
 
 	memcg_release_pages(cachep, cachep->gfporder);
 	if (current->reclaim_state)
mm/slab.h renamed from mm/slab.h~slub-reset-page-at-free +11 -0
diff -puN mm/slab.h~slub-reset-page-at-free mm/slab.h
@@ -279,3 +279,14 @@ struct kmem_cache_node {
 void *slab_next(struct seq_file *m, void *p, loff_t *pos);
 void slab_stop(struct seq_file *m, void *p);
 
+/*
+ * The slab allocators use 'struct page' fields for all kinds of
+ * things.  This resets the page so that the buddy allocator will
+ * be happy with it.
+ */
+static inline void slab_reset_page(struct page *page)
+{
+	page->mapping = NULL;
+	page_mapcount_reset(page);
+}
+
mm/slob.c renamed from mm/slob.c~slub-reset-page-at-free +1 -1
diff -puN mm/slob.c~slub-reset-page-at-free mm/slob.c
@@ -371,7 +371,7 @@ static void slob_free(void *block, int s
 			clear_slob_page_free(sp);
 		spin_unlock_irqrestore(&slob_lock, flags);
 		__ClearPageSlab((struct page *)sp);
-		page_mapcount_reset((struct page *)sp);
+		slab_reset_page((struct page *)sp);
 		slob_free_pages(b, 0);
 		return;
 	}
mm/slub.c renamed from mm/slub.c~slub-reset-page-at-free +1 -1
diff -puN mm/slub.c~slub-reset-page-at-free mm/slub.c
@@ -1530,7 +1530,7 @@ static void __free_slab(struct kmem_cach
 	__ClearPageSlab(page);
 
 	memcg_release_pages(s, order);
-	page_mapcount_reset(page);
+	slab_reset_page(page);
 	if (current->reclaim_state)
 		current->reclaim_state->reclaimed_slab += pages;
 	__free_memcg_kmem_pages(page, order);

_

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#8 lore raw

Re: [RFC][PATCH 3/3] mm: slabs: reset page at free

Quoted 9 lines · Dave Hansen

We now have slub's ->freelist usage impinging on page->mapping's storage space. The buddy allocator wants ->mapping to be NULL when a page is handed back, so we have to make sure that it is cleared.

Note that slab already doeds this, so just create a common helper and have all the slabs do it this way. ->mapping is right next to ->flags, so it's virtually guaranteed to be in the L1 at this point, so this shouldn't cost very much to do in practice.

Maybe add a common page alloc and free function in mm/slab_common.c?

All the allocators do similar things after all.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>