SLAB : [NUMA] keep nodeid in struct page instead of struct slab
RFC PATCH 1/1 SLAB : [NUMA] keep nodeid in struct page instead of struct slab
In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.
1) MAX_NUMNODES <= 64
2) alignment of 'struct kmem_cache *' can be >= 64
The following patch changes the page->lru.next to contain not only the 'struct kmem_cache *' pointer, but also the nodeid in the low order bits.
This also reduces sizeof(struct slab) by 8 bytes on 64bits arches. This reduces sizeof(struct slab) on all platforms (UP, or SMP)
mm/slab.c +49 -15
@@ -210,6 +210,16 @@ #define BUFCTL_FREE (((kmem_bufctl_t)(~0#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3) +#ifdef CONFIG_NUMA +#if MAX_NUMNODES <=64 +/* we can use low order bits of page->lru.next to store nodeids */ +# define KEEP_NODEID_IN_PAGE +#else +/* too many nodes, we need a field in 'struct slab' */ +# define KEEP_NODEID_IN_SLAB +#endif +#endif +/* * struct slab *
@@ -223,7 +233,9 @@ struct slab {void *s_mem;/* including colour offset */ unsigned int inuse;/* num of objs active in slab */ kmem_bufctl_t free; +#ifdef KEEP_NODEID_IN_SLAB unsigned short nodeid; +#endif };/*
@@ -585,9 +597,18 @@ static int slab_break_gfp_order = BREAK_* allocator. These are used to find the slab an obj belongs to. With kfree(), * these are used to find the cache which an obj belongs to. */ -static inline void page_set_cache (struct page *page,struct kmem_cache *cache) +static inline void page_set_cache_slab_nodeid (struct page *page, +struct kmem_cache *cache,struct slab *slab,int nodeid) { + page->lru.prev = (struct list_head *)slab; +#ifdef KEEP_NODEID_IN_PAGE + page->lru.next = (struct list_head *)((long )cache + nodeid); +#else page->lru.next = (struct list_head *)cache; +#endif +#ifdef KEEP_NODEID_IN_SLAB + slab->nodeid = nodeid; +#endif }static inline struct kmem_cache *page_get_cache (struct page *page)
@@ -595,12 +616,11 @@ static inline struct kmem_cache *page_geif (unlikely (PageCompound (page))) page = (struct page *)page_private (page);BUG_ON (!PageSlab (page)); +#ifdef KEEP_NODEID_IN_PAGE +return (struct kmem_cache *)((long )page->lru.next & ~63 ); +#else return (struct kmem_cache *)page->lru.next; -} - -static inline void page_set_slab (struct page *page,struct slab *slab) -{ - page->lru.prev = (struct list_head *)slab; +#endif }static inline struct slab *page_get_slab (struct page *page)
@@ -617,6 +637,18 @@ static inline struct kmem_cache *virt_toreturn page_get_cache (page); } +#ifdef CONFIG_NUMA +static inline int virt_to_nodeid (const void *obj) +{ +struct page *page =virt_to_page (obj); +#ifdef KEEP_NODEID_IN_SLAB +return page_get_slab (page)->nodeid; +#else +return (long )page->lru.next &63 ; +#endif +} +#endif +static inline struct slab *virt_to_slab (const void *obj) {struct page *page =virt_to_page (obj);
@@ -1134,8 +1166,7 @@ static void drain_alien_cache(struct kmestatic inline int cache_free_alien (struct kmem_cache *cachep,void *objp) { -struct slab *slabp =virt_to_slab (objp); -int nodeid = slabp->nodeid; +int nodeid =virt_to_nodeid (objp);struct kmem_list3 *l3;struct array_cache *alien =NULL ;int node;
@@ -1146,7 +1177,7 @@ static inline int cache_free_alien(struc* Make sure we are not freeing a object from another node to the array * cache on this cpu. */ -if (likely (slabp->nodeid == node) ||unlikely (!use_alien_caches)) +if (likely (nodeid == node) ||unlikely (!use_alien_caches))return 0 ; l3 = cachep->nodelists[node];
@@ -1437,8 +1468,14 @@ void __init kmem_cache_init(void) cache_cache.array[smp_processor_id ()] = &initarray_cache.cache; cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE ]; +#ifdef KEEP_NODEID_IN_PAGE +/* kmem_cache addresses must be multiple of 64 */ + cache_cache.buffer_size =ALIGN (cache_cache.buffer_size, +max (64 ,cache_line_size ())); +#else cache_cache.buffer_size =ALIGN (cache_cache.buffer_size,cache_line_size ()); +#endif cache_cache.reciprocal_buffer_size =reciprocal_value (cache_cache.buffer_size);
@@ -2588,7 +2625,6 @@ static struct slab *alloc_slabmgmt(struc slabp->inuse =0 ; slabp->colouroff = colour_off; slabp->s_mem = objp + colour_off; - slabp->nodeid = nodeid;return slabp; }
@@ -2699,7 +2735,7 @@ #endif* virtual address for kfree, ksize, kmem_ptr_validate, and slab debugging. */ static void slab_map_pages (struct kmem_cache *cache,struct slab *slab, -void *addr) +void *addr,int nodeid) {int nr_pages;struct page *page;
@@ -2711,8 +2747,7 @@ static void slab_map_pages(struct kmem_c nr_pages <<= cache->gfporder;do { -page_set_cache (page, cache); -page_set_slab (page, slab); +page_set_cache_slab_nodeid (page, cache, slab, nodeid); page++; }while (--nr_pages); }
@@ -2787,8 +2822,7 @@ static int cache_grow(struct kmem_cacheif (!slabp) goto opps1; - slabp->nodeid = nodeid; -slab_map_pages (cachep, slabp, objp); +slab_map_pages (cachep, slabp, objp, nodeid);cache_init_objs (cachep, slabp, ctor_flags);
Quoted 2 lines · Eric Dumazet
In order to avoid a cache miss in
kmem_cache_free()on NUMA and reduce hot path length, we could exploit the following common facts.
It would be nice if you could cc me for slab patches.
mm/slab.c static int slab_break_gfp_order = BREAK_ · line 611
Quoted 13 lines · Eric Dumazet
mm/slab.c +10
-1-static inline void page_set_cache (struct page *page,struct kmem_cache *cache) +static inline void page_set_cache_slab_nodeid (struct page *page, +struct kmem_cache *cache,struct slab *slab,int nodeid) { + page->lru.prev = (struct list_head *)slab; +#ifdef KEEP_NODEID_IN_PAGE + page->lru.next = (struct list_head *)((long )cache + nodeid); +#else page->lru.next = (struct list_head *)cache; +#endif +#ifdef KEEP_NODEID_IN_SLAB + slab->nodeid = nodeid; +#endif
Can we please have a slab_get_nid() and slab_set_nid() instead which reduces the need for #ifdefs?
Quoted 2 lines · Pekka Enberg
Can we please have a
slab_get_nid()andslab_set_nid()instead which reduces the need for #ifdefs?
None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.
Sounds good. Do you know for a fact that we don't or are you just afraid we have a bug?
Quoted 7 lines · Pekka Enberg
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.Sounds good. Do you know for a fact that we don't or are you just afraid we have a bug?
I tried to clean it up during the GFP_THISNODE work but there was some issue on powerpc where we needed fallback for bootstrap. There may be more trouble there. Checking the allocs where we do not use GFP_THISNODE should get you there.
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.
Oh well, here's a patch to find out. (I don't have a NUMA box.)
Pekka
From: Pekka Enberg <[email protected]>
This adds a virt_to_nodeid() for looking up the NUMA node ID for a slab object. The current implementation uses slab->nodeid but adds a WARN_ON to catch cases where page nid differs. Eventually, we can change virt_to_nodeid() to use page_to_nid after which slab->nodeid can be removed.
--- mm/slab.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)
mm/slab.c +12 -5
@@ -623,6 +623,14 @@ static inline struct slab *virt_to_slab(return page_get_slab (page); } +static inline unsigned short virt_to_nodeid (const void *obj) +{ +struct page *page =virt_to_page (obj); +struct slab *slab =page_get_slab (page); +WARN_ON (slab->nodeid !=page_to_nid (page)); +return slab->nodeid; +} +static inline void *index_to_obj (struct kmem_cache *cache,struct slab *slab,unsigned int idx) {
@@ -1134,8 +1142,7 @@ int i = 0;static inline int cache_free_alien (struct kmem_cache *cachep,void *objp) { -struct slab *slabp =virt_to_slab (objp); -int nodeid = slabp->nodeid; +int nodeid =virt_to_nodeid (objp);struct kmem_list3 *l3;struct array_cache *alien =NULL ;int node;
@@ -1146,7 +1153,7 @@ static inline int cache_free_alien(struc* Make sure we are not freeing a object from another node to the array * cache on this cpu. */ -if (likely (slabp->nodeid == node) ||unlikely (!use_alien_caches)) +if (likely (nodeid == node) ||unlikely (!use_alien_caches))return 0 ; l3 = cachep->nodelists[node];
@@ -2665,7 +2672,7 @@ static void *slab_get_obj(struct kmem_ca next =slab_bufctl (slabp)[slabp->free];#if DEBUG slab_bufctl (slabp)[slabp->free] =BUFCTL_FREE ; -WARN_ON (slabp->nodeid != nodeid); +WARN_ON (virt_to_nodeid (objp) != nodeid);#endif slabp->free = next;
@@ -2679,7 +2686,7 @@ static void slab_put_obj(struct kmem_cac#if DEBUG /* Verify that the slab belongs to the intended node */ -WARN_ON (slabp->nodeid != nodeid); +WARN_ON (virt_to_nodeid (objp) != nodeid);if (slab_bufctl (slabp)[objnr] +1 <=SLAB_LIMIT +1 ) { printk(KERN_ERR "slab: double free detected in cache "
Quoted 7 lines · On Wed, 21 Mar 2007 17:42:22 +0200 (EET) Pekka J Enberg…
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.Oh well, here's a patch to find out. (I don't have a NUMA box.)
This wont work, because of use_alien_caches knob.
If use_alien_caches is set to 0, your WARN_ON will trigger many false alarms
Quoted 4 lines · Eric Dumazet
This wont work, because of
use_alien_cachesknob.If
use_alien_cachesis set to 0, yourWARN_ONwill trigger many false alarms
How is that? The mapping should always be the same regardless of whether we use alien caches or not.
Quoted 6 lines · Pekka J Enberg
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.Oh well, here's a patch to find out. (I don't have a NUMA box.)
You can always use numa emulation on x86-64. Boot with numa=fake=NODES
-Andi
Quoted 2 lines · Andi Kleen
You can always use numa emulation on x86-64. Boot with numa=fake=NODES
Yes, I know, but I don't have a x86-64 box either =)
Quoted 9 lines · Andi Kleen
Quoted 6 lines · Pekka J Enberg
Quoted 3 lines · Christoph Lameter
None of that please. The page flags already contain a node number that is accessible via
page_to_nid(). Just make sure that we never put a slab onto the wrong node.Oh well, here's a patch to find out. (I don't have a NUMA box.)
You can always use numa emulation on x86-64. Boot with numa=fake=NODES
This wont help you to debug strange fallback scenarios on unusual NUMA platforms.
[RFC] NUMA : could we introduce virt_to_nid() ?
In reply to Andi KleenHi Andi
Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.
/* common implementation *
int virt_to_nid(const void *addr)
{
struct page *page = virt_to_page(addr);
return page_to_nid(page);
}On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?
Thank you
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
Quoted 5 lines · Eric Dumazet
Checking Christoph quicklist implementation, I found the same cache miss in
free()than SLAB has./* common implementation *
intvirt_to_nid(const void *addr)
Quoted 7 lines
{ struct page *page = virt_to_page(addr); return page_to_nid(page); }On some platforms (
x86_64for example), could we have a better implementation, not accessing struct page, but usingphys_to_nid()?
Sounds good to me. At least cache_free_alien() in mm/slab.c to should be converted to use it.
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
Quoted 11 lines · On Fri, 23 Mar 2007 14:48:24 +0200 "Pekka Enberg" <[email protected]>…
Quoted 10 lines · Eric Dumazet
Checking Christoph quicklist implementation, I found the same cache miss in
free()than SLAB has./* common implementation *
intvirt_to_nid(const void *addr){ struct page *page = virt_to_page(addr); return page_to_nid(page); }
Quoted 6 lines
Quoted 2 lines
On some platforms (
x86_64for example), could we have a better implementation, not accessing struct page, but usingphys_to_nid()?Sounds good to me. At least
cache_free_alien()inmm/slab.cto should be converted to use it.
Not yet :(
Because in slab we currently need virt_to_slab(addr)->nodeid, not page_to_nid(virt_to_page(addr))
It might be different, according to Christoph
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
In reply to Eric DumazetQuoted 4 lines · Eric Dumazet
Hi Andi
Checking Christoph quicklist implementation, I found the same cache miss in
free()than SLAB has.
Quoted 8 lines
/* common implementation *
intvirt_to_nid(const void *addr){ struct page *page = virt_to_page(addr); return page_to_nid(page); }On some platforms (
x86_64for example), could we have a better implementation, not accessing struct page, but usingphys_to_nid()?
Sure why not. Please send a patch.
-Andi
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
In reply to Eric DumazetQuoted 11 lines · Eric Dumazet
Checking Christoph quicklist implementation, I found the same cache miss in
free()than SLAB has./* common implementation *
intvirt_to_nid(const void *addr){ struct page *page = virt_to_page(addr); return page_to_nid(page); }On some platforms (
x86_64for example), could we have a better implementation, not accessing struct page, but usingphys_to_nid()?
This is going to pollute the caches since we then use multiple ways to determine the node of a page. Its better to stay with the same approach for all pages. The page struct is used for many other purposes as well. Its likely to be in the cpu cache.
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
Quoted 11 lines · On Fri, 23 Mar 2007 07:50:28 -0700 (PDT) Christoph Lameter…
Quoted 9 lines · Eric Dumazet
Checking Christoph quicklist implementation, I found the same cache miss in
free()than SLAB has./* common implementation *
intvirt_to_nid(const void *addr){ struct page *page = virt_to_page(addr); return page_to_nid(page); }
Quoted 8 lines
Quoted 2 lines
On some platforms (
x86_64for example), could we have a better implementation, not accessing struct page, but usingphys_to_nid()?This is going to pollute the caches since we then use multiple ways to determine the node of a page. Its better to stay with the same approach for all pages. The page struct is used for many other purposes as well. Its likely to be in the cpu cache.
Sorry ? page structs are not in cpu cache at all.
phys_to_nid() on my 16 GB x86_64 machine uses one single cache line, shared by all pages. This single cache line is cache hot yes, not "struct page"s ...
And on this machine, thats about 224 Mbytes of 'struct pages'
You carefully commented your alloc() function saying it is touching two cache lines. But you omited to say that free() function needs 3 cache lines if CONFIG_NUMA
For SLAB use, page struct is needed because we use lru.{next|prev} to store slab/cachep pointers, but for a pure page allocator, unless I misread your patch, we dont need it, if virt_to_nid() can do its job without it.
Re: [RFC] NUMA : could we introduce virt_to_nid() ?
Quoted 1 line · Eric Dumazet
Sorry ? page structs are not in cpu cache at all.
They are if they are in any way handled by the VM.
Quoted 2 lines
You carefully commented your
alloc()function saying it is touching two cache lines. But you omited to say thatfree()function needs 3 cache lines ifCONFIG_NUMA
Yes it needs the third cacheline to check the node of the page in the NUMA case.
Quoted 4 lines
For SLAB use, page struct is needed because we use lru.{next|prev} to store slab/cachep pointers, but for a pure page allocator, unless I misread your patch, we dont need it, if
virt_to_nid()can do its job without it.
The page allocator is only handling page_structs.
mm/slab.c void __init kmem_cache_init(void) · line 1478
Quoted 8 lines · Eric Dumazet
mm/slab.c +6
-0+#ifdef KEEP_NODEID_IN_PAGE +/* kmem_cache addresses must be multiple of 64 */ + cache_cache.buffer_size =ALIGN (cache_cache.buffer_size, +max (64 ,cache_line_size ())); +#else cache_cache.buffer_size =ALIGN (cache_cache.buffer_size,cache_line_size ()); +#endif
Please introduce cache_cache_align() to hide the #ifdeffing.
Quoted 3 lines · Eric Dumazet
In order to avoid a cache miss in
kmem_cache_free()on NUMA and reduce hot path length, we could exploit the following common facts.1)
MAX_NUMNODES<= 64
MAX_NUMNODES == 1024 on IA64 and may be on x86_64 in the future.
Quoted 5 lines
2) alignment of 'struct
kmem_cache*' can be >= 64The following patch changes the
page->lru.nextto contain not only the 'structkmem_cache*' pointer, but also the nodeid in the low order bits.
Nack.
I would suggest you audit slab and make sure that we never fall back and always enter a slab on the node of page_to_nid(page). If
all allocations use GFP_THISNODE then you should be safe and then we can remove the nodeid from the slab structure and simply use the available node information in the page struct.
Quoted 7 lines · On Wed, 21 Mar 2007 07:41:46 -0700 (PDT) Christoph Lameter…
Quoted 3 lines · Eric Dumazet
In order to avoid a cache miss in
kmem_cache_free()on NUMA and reduce hot path length, we could exploit the following common facts.1)
MAX_NUMNODES<= 64
MAX_NUMNODES== 1024 on IA64 and may be onx86_64in the future.
So what ? I am very happy for you Christoph.
Average linux machines (ie more than 99.99 % on this planet) are using MAX_NUMNODES <= 64
If you read my patch, your lovely 1024 nodes machines will run as before : No change at all for big machines.
Quoted 7 lines
Quoted 6 lines
2) alignment of 'struct
kmem_cache*' can be >= 64The following patch changes the
page->lru.nextto contain not only the 'structkmem_cache*' pointer, but also the nodeid in the low order bits.
Quoted 8 lines
Nack.
I would suggest you audit slab and make sure that we never fall back and always enter a slab on the node of
page_to_nid(page). If
all allocations useGFP_THISNODEthen you should be safe and then we can remove the nodeid from the slab structure and simply use the available node information in the page struct.
Last time I checked 'struct page', they was no nodeid in it.
I cooked this patch because you told me it was possible to have a 'slab nodeid' different than page_to_nid(obj). Now you want me to prove the contrary. Me confused.
Quoted 5 lines · Eric Dumazet
Quoted 2 lines · On Wed, 21 Mar 2007 07:41:46 -0700 (PDT) Christoph Lameter…
On Wed, 21 Mar 2007, Eric Dumazet wrote:
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
In order to avoid a cache miss in
kmem_cache_free()on NUMA and reduce hot path length, we could exploit the following common facts.1)
MAX_NUMNODES<= 64
MAX_NUMNODES== 1024 on IA64 and may be onx86_64in the future.So what ? I am very happy for you Christoph.
You wanted to exploit that MAX_NUMNODES <= 64?
Quoted 1 line · On Wed, 21 Mar 2007 08:44:53 -0700 (PDT) Christoph Lameter…
You wanted to exploit that
MAX_NUMNODES<= 64?
Maybe my english is not perfect. I promise I improve it eventually.
My patch was for unlucky guys that have MAX_NUMNODES <= 64.
For others, the preprocessor gave the old code.
Quoted 1 line · Eric Dumazet
Last time I checked 'struct page', they was no nodeid in it.
Hmm, page_to_nid() in include/linx/mm.h doesn't agree with you:
#ifdef NODE_NOT_IN_PAGE_FLAGS
extern int page_to_nid(struct page *page);
#else
static inline int page_to_nid(struct page *page)
{
return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
}#endif
Quoted 5 lines · On Wed, 21 Mar 2007 17:47:13 +0200 "Pekka Enberg" <[email protected]>…
Quoted 1 line · Eric Dumazet
Last time I checked 'struct page', they was no nodeid in it.
Hmm,
page_to_nid()ininclude/linx/mm.hdoesn't agree with you:
Quoted 9 lines
#ifdef
NODE_NOT_IN_PAGE_FLAGS
extern intpage_to_nid(struct page *page);
#else
static inline intpage_to_nid(struct page *page){ return (page->flags >> NODES_PGSHIFT) & NODES_MASK; }#endif
Correct, but still another ifdef NODE_NOT_IN_PAGE_FLAGS
Anyway the correct path is to check GFP_THISNODE stuff, because we can even avoid access to 'struct page' to get the nodeid, at least on x86_64 NUMA, where struct memnode can provide the information with less cache footprint.
Quoted 8 lines · Eric Dumazet
In order to avoid a cache miss in
kmem_cache_free()on NUMA and reduce hot path length, we could exploit the following common facts.1)
MAX_NUMNODES<= 642) alignment of 'struct
kmem_cache*' can be >= 64The following patch changes the
page->lru.nextto contain not only the 'structkmem_cache*' pointer, but also the nodeid in the low order bits.
It might be cleaner to just put it into flags on 64bit. There is enough space there and we don't really care about 32bit NUMA machines.
-Andi