mm: slab: separate slab_page from 'struct page'
Subsystems (3)
RFC PATCH 1/1 mm: slab: separate slab_page from 'struct page'
This is a major RFC. *SHOULD* we do something like this?
'struct page' is a 140-line crime against humanity at this point. Most of the blame for its girth lies at the feet of the slab allocators. While the effort at reuse of unused 'struct page' fields is laudable, the result is:
struct page {
struct {
union {
struct {
union {
struct {
};
};
};
};
};
union {
struct {
};
};
union {
};
}What the slab allocators really want is a new structure that shares very little with 'struct page'. So, let's give that to
them: 'struct slab_page'. As far as I can tell, page-flags is just about the only thing in 'struct page' that the slab code really cares about. As long as flags in 'slab_page' and 'struct page' line up, we should be able to cast pointers back and forth without anything awful happening. Note: I even enforce this sharing with a BUILD_BUG_ON(), so this *can* be as safe the unions were.
At least for slab, this doesn't turn out to be too big of a deal:
it's only 8 casts. slub looks like it'll be a bit more work, but still manageable.
Note: there are a couple more patches if you want to actually run this. page->lru vs. page->list needs to get resolved too.
---
linux.git-davehans/include/linux/mm_types.h | 169 ++++++++++++++-------------- linux.git-davehans/mm/slab.c | 104 ++++++++++------- linux.git-davehans/mm/slab.h | 4 3 files changed, 155 insertions(+), 122 deletions(-)
include/linux/mm_types.h renamed from include/linux/mm_types.h~make-separate-slab-page +90 -79
@@ -44,93 +44,35 @@ struct page {/* First double word block */ unsigned long flags;/* Atomic flags, some possibly * updated asynchronously */ -union { -struct address_space *mapping;/* If low bit clear, points to -* inode address_space, or NULL. -* If page mapped as anonymous -* memory, low bit is set, and -* it points to anon_vma object: -* see PAGE_MAPPING_ANON below. -*/ -void *s_mem;/* slab first object */ - }; +struct address_space *mapping;/* If low bit clear, points to +* inode address_space, or NULL. +* If page mapped as anonymous +* memory, low bit is set, and +* it points to anon_vma object: +* see PAGE_MAPPING_ANON below. +*/ /* Second double word */ -struct { -union { -pgoff_t index;/* Our offset within mapping. */ -void *freelist;/* sl[aou]b first free object */ - }; - -union { -#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE ) && \ - defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE ) -/* Used for cmpxchg_double in slub */ -unsigned long counters; -#else -/* -* Keep _count separate from slub cmpxchg_double data. -* As the rest of the double word is protected by -* slab_lock but _count is not. -*/ -unsigned counters; -#endif - -struct { - -union { -/* -* Count of ptes mapped in -* mms, to show when page is -* mapped & limit reverse map -* searches. -* -* Used also for tail pages -* refcounting instead of -* _count. Tail pages cannot -* be mapped and keeping the -* tail page _count zero at -* all times guarantees -* get_page_unless_zero() will -* never succeed on tail -* pages. -*/ -atomic_t _mapcount; +pgoff_t index;/* Our offset within mapping. */ -struct {/* SLUB */ -unsigned inuse:16 ; -unsigned objects:15 ; -unsigned frozen:1 ; - }; -int units;/* SLOB */ - }; -atomic_t _count;/* Usage count, see below. */ - }; -unsigned int active;/* SLAB */ - }; - }; +/* +* Count of ptes mapped in mms, to show when page is +* mapped & limit reverse map searches. +* +* Used also for tail pages refcounting instead of +* _count. Tail pages cannot be mapped and keeping the +* tail page _count zero at all times guarantees +* get_page_unless_zero() will never succeed on tail +* pages. +*/ +atomic_t _mapcount; +atomic_t _count;/* Usage count, see below. */ /* Third double word block */ union {struct list_head lru;/* Pageout list, eg. active_list * protected by zone->lru_lock ! */ -struct {/* slub per cpu partial pages */ -struct page *next;/* Next partial slab */ -#ifdef CONFIG_64BIT -int pages;/* Nr of partial slabs left */ -int pobjects;/* Approximate # of objects */ -#else -short int pages; -short int pobjects; -#endif - }; - -struct list_head list;/* slobs list of pages */ -struct slab *slab_page;/* slab fields */ -struct rcu_head rcu_head;/* Used by SLAB -* when destroying via RCU -*/ #if defined(CONFIG_TRANSPARENT_HUGEPAGE ) &&USE_SPLIT_PMD_PTLOCKS pgtable_t pmd_huge_pte;/* protected by page->ptl */ #endif
@@ -152,7 +94,6 @@ struct page {spinlock_t ptl;#endif #endif -struct kmem_cache *slab_cache;/* SL[AU]B: Pointer to slab */ struct page *first_page;/* Compound tail pages */ };
@@ -188,6 +129,76 @@ struct page { }/* * The struct page can be forced to be double word aligned so that atomic ops +* on double words work. The SLUB allocator can make use of such a feature. +*/ +#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE +__aligned(2 * sizeof(unsigned long)) +#endif +; + +struct slab_page { +/* First double word block */ +unsigned long flags;/* Atomic flags, some possibly +* updated asynchronously */ +void *s_mem;/* slab first object */ + +/* Second double word */ +void *freelist;/* sl[aou]b first free object */ + +union { +#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE ) && \ + defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE ) +/* Used for cmpxchg_double in slub */ +unsigned long counters; +#else +/* +* Keep _count separate from slub cmpxchg_double data. +* As the rest of the double word is protected by +* slab_lock but _count is not. +*/ +unsigned counters; +#endif + +struct { + +union { +struct {/* SLUB */ +unsigned inuse:16 ; +unsigned objects:15 ; +unsigned frozen:1 ; + }; +int units;/* SLOB */ + }; + }; +unsigned int active;/* SLAB */ + }; + +/* Third double word block */ +union { +struct {/* slub per cpu partial pages */ +struct page *next;/* Next partial slab */ +#ifdef CONFIG_64BIT +int pages;/* Nr of partial slabs left */ +int pobjects;/* Approximate # of objects */ +#else +short int pages; +short int pobjects; +#endif + }; + +struct list_head list;/* slobs list of pages */ +struct slab *slab_page;/* slab fields */ +struct rcu_head rcu_head;/* Used by SLAB +* when destroying via RCU +*/ + }; + +/* Remainder is not double word aligned */ +struct kmem_cache *slab_cache;/* SL[AU]B: Pointer to slab */ + +} +/* +* The struct page can be forced to be double word aligned so that atomic ops * on double words work. The SLUB allocator can make use of such a feature. */ #ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
mm/slab.c renamed from mm/slab.c~make-separate-slab-page +63 -41
332 lines not on this page · raw message
mm/slab.h renamed from mm/slab.h~make-separate-slab-page +2 -2
@@ -220,7 +220,7 @@ static inline struct kmem_cache *memcg_rstatic inline struct kmem_cache *cache_from_obj (struct kmem_cache *s,void *x) {struct kmem_cache *cachep; -struct page *page; +struct slab_page *page;/* * When kmemcg is not being used, both assignments should return the
@@ -232,7 +232,7 @@ static inline struct kmem_cache *cache_fif (!memcg_kmem_enabled () && !unlikely (s->flags &SLAB_DEBUG_FREE ))return s; - page =virt_to_head_page (x); + page = (struct slab_page *)virt_to_head_page (x); cachep = page->slab_cache;if (slab_equal_or_root (cachep, s))return cachep;
_
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 3 lines · Dave Hansen
At least for slab, this doesn't turn out to be too big of a deal:
it's only 8 casts. slub looks like it'll be a bit more work, but still manageable.
The single page struct definitions makes it easy to see how a certain field is being used in various subsystems. If you add a field then you can see other use cases in other subsystems. If you happen to call them then you know that there is trouble afoot.
Also if you screw up the sizes then you screw up the page struct for everything and its very evident that a problem exists.
How do you ensure that the sizes and the locations of the fields in multiple page structs stay consistent?
As far as I can tell we are trying to put everything into one page struct to keep track of the uses of various fields and to allow a reference for newcomes to the kernel.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 9 lines · Christoph Lameter
Quoted 3 lines · Dave Hansen
At least for slab, this doesn't turn out to be too big of a deal:
it's only 8 casts. slub looks like it'll be a bit more work, but still manageable.The single page struct definitions makes it easy to see how a certain field is being used in various subsystems. If you add a field then you can see other use cases in other subsystems. If you happen to call them then you know that there is trouble afoot.
First of all, I'd really argue with the assertion that the way it is now make it easy to figure anything out. Maybe we can take a vote. :)
We _need_ to share fields when the structure is handed between different subsystems and it needs to be consistent in both places. For slab page at least, the only data that actually gets used consistently is page->flags. It seems silly to bend over backwards just to share a single bitfield.
Quoted 2 lines
How do you ensure that the sizes and the locations of the fields in multiple page structs stay consistent?
Check out the BUILD_BUG_ON(). That shows one example of how we do it for a field location. We could do the same for sizeof() the two.
Quoted 3 lines
As far as I can tell we are trying to put everything into one page struct to keep track of the uses of various fields and to allow a reference for newcomes to the kernel.
If the goal is to make a structure which is approachable to newcomers to the kernel, then I think we've utterly failed.
Quoted 8 lines · Dave Hansen
Quoted 5 lines
The single page struct definitions makes it easy to see how a certain field is being used in various subsystems. If you add a field then you can see other use cases in other subsystems. If you happen to call them then you know that there is trouble afoot.
First of all, I'd really argue with the assertion that the way it is now make it easy to figure anything out. Maybe we can take a vote. :)
Its certainly easier than it was before where we had page struct defs spluttered in various subsystems.
Quoted 5 lines
We _need_ to share fields when the structure is handed between different subsystems and it needs to be consistent in both places. For slab page at least, the only data that actually gets used consistently is
page->flags. It seems silly to bend over backwards just to share a single bitfield.
If you get corruption in one field then you need to figure out which other subsystem could have accessed that field. Its not a single bitfield. There are numerous relationships between the fields in struct page.
Quoted 5 lines
Quoted 2 lines
How do you ensure that the sizes and the locations of the fields in multiple page structs stay consistent?
Check out the
BUILD_BUG_ON(). That shows one example of how we do it for a field location. We could do the same forsizeof()the two.
A bazillion of those? And this is simpler than what we ahve?
Quoted 6 lines
Quoted 3 lines
As far as I can tell we are trying to put everything into one page struct to keep track of the uses of various fields and to allow a reference for newcomes to the kernel.
If the goal is to make a structure which is approachable to newcomers to the kernel, then I think we've utterly failed.
I do not see your approach making things easier. Having this stuff in one place is helpful. I kept on discovering special use cases in various kernel subsystems that caused breakage because of this and that special use cases for fields. I think we were only able to optimize slabs use of struct page because we finally had a better handle on what uses which field for what purpose.
Looks to me that you want to go back to the old mess because we now have a more complete accounting of how the fields are used. It may be a horror but maybe you can help by simplifying things where possible and find as of yet undocumented use cases for various page struct fields?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · Christoph Lameter
Quoted 5 lines
Quoted 5 lines
We _need_ to share fields when the structure is handed between different subsystems and it needs to be consistent in both places. For slab page at least, the only data that actually gets used consistently is
page->flags. It seems silly to bend over backwards just to share a single bitfield.If you get corruption in one field then you need to figure out which other subsystem could have accessed that field. Its not a single bitfield. There are numerous relationships between the fields in struct page.
I'm not saying that every 'struct page' user should get their own complete structure. I'm just saying that the *slabs* should get their own structure. Let's go through it field by field for the "normal" 'struct page' without debugging options:
page->flags: shared by everybody, needs to be consistent for things
like memory error handling
mapping: unioned over by s_mem for slab
index: unioned over by freelist for sl[oua]b_count: unioned over by lots of stuff by sl[oua]b
lru: unioned over by lots of stuff by sl[oua]b, including another
list_head called 'list' which blk-mq.c is now using.
private: opaque storage anyway, but unioned over by sl[au]b
See? *EVERYTHING* is overridden by at least one of the sl?b allocators except ->flags. In other words, there *ARE* no relationships when it comes to the sl?bs, except for page->flags.
Quoted 3 lines · Dave Hansen
See? *EVERYTHING* is overridden by at least one of the sl?b allocators except ->flags. In other words, there *ARE* no relationships when it comes to the sl?bs, except for
page->flags.
Slab objects can be used for I/O and then the page fields become important.
Quoted 7 lines · Christoph Lameter
Quoted 3 lines · Dave Hansen
See? *EVERYTHING* is overridden by at least one of the sl?b allocators except ->flags. In other words, there *ARE* no relationships when it comes to the sl?bs, except for
page->flags.Slab objects can be used for I/O and then the page fields become important.
OK, which fields? How are they important? Looking at 'struct page', I don't see any fields other than ->flags that the slab allocators leave alone.
I do see some refcounting (page->_count) done on request_queue->dma_drain_buffer which is kmalloc()'d. Although, I'm a bit skeptical that this is correct or has been audited because the slabs do seem to write over this storage. Anything needing to do per-page could probably be converted over to a raw alloc_pages() anyway because it couldn't possibly be doing sub-page-size allocations.
What other cases were you thinking of?