Skip to contentSkip to conversation

SLAB : [NUMA] keep nodeid in struct page instead of struct slab

RFC PATCH NAK Eric Dumazet 4 participants 26 messages Mar 21, 2007 – Mar 23, 2007 Memory management

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 SLAB : [NUMA] keep nodeid in struct page instead of struct slab

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

1) MAX_NUMNODES <= 64

2) alignment of 'struct kmem_cache *' can be >= 64

The following patch changes the page->lru.next to contain not only the 'struct kmem_cache *' pointer, but also the nodeid in the low order bits.

This also reduces sizeof(struct slab) by 8 bytes on 64bits arches. This reduces sizeof(struct slab) on all platforms (UP, or SMP)

mm/slab.c +49 -15
index abf46ae..d2f7299 100644
@@ -210,6 +210,16 @@ #define BUFCTL_FREE	(((kmem_bufctl_t)(~0
 #define	BUFCTL_ACTIVE	(((kmem_bufctl_t)(~0U))-2)
 #define	SLAB_LIMIT	(((kmem_bufctl_t)(~0U))-3)
 
+#ifdef CONFIG_NUMA
+#if MAX_NUMNODES <= 64
+/* we can use low order bits of page->lru.next to store nodeids */
+# define KEEP_NODEID_IN_PAGE
+#else
+/* too many nodes, we need a field in 'struct slab' */
+# define KEEP_NODEID_IN_SLAB
+#endif
+#endif
+
 /*
  * struct slab
  *
@@ -223,7 +233,9 @@ struct slab {
 	void *s_mem;		/* including colour offset */
 	unsigned int inuse;	/* num of objs active in slab */
 	kmem_bufctl_t free;
+#ifdef KEEP_NODEID_IN_SLAB
 	unsigned short nodeid;
+#endif
 };
 
 /*
@@ -585,9 +597,18 @@ static int slab_break_gfp_order = BREAK_
  * allocator.  These are used to find the slab an obj belongs to.  With kfree(),
  * these are used to find the cache which an obj belongs to.
  */
-static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
+static inline void page_set_cache_slab_nodeid(struct page *page,
+	struct kmem_cache *cache, struct slab *slab, int nodeid)
 {
+	page->lru.prev = (struct list_head *)slab;
+#ifdef KEEP_NODEID_IN_PAGE
+	page->lru.next = (struct list_head *)((long)cache + nodeid);
+#else
 	page->lru.next = (struct list_head *)cache;
+#endif
+#ifdef KEEP_NODEID_IN_SLAB
+	slab->nodeid = nodeid;
+#endif
 }
 
 static inline struct kmem_cache *page_get_cache(struct page *page)
@@ -595,12 +616,11 @@ static inline struct kmem_cache *page_ge
 	if (unlikely(PageCompound(page)))
 		page = (struct page *)page_private(page);
 	BUG_ON(!PageSlab(page));
+#ifdef KEEP_NODEID_IN_PAGE
+	return (struct kmem_cache *)((long)page->lru.next & ~63);
+#else
 	return (struct kmem_cache *)page->lru.next;
-}
-
-static inline void page_set_slab(struct page *page, struct slab *slab)
-{
-	page->lru.prev = (struct list_head *)slab;
+#endif
 }
 
 static inline struct slab *page_get_slab(struct page *page)
@@ -617,6 +637,18 @@ static inline struct kmem_cache *virt_to
 	return page_get_cache(page);
 }
 
+#ifdef CONFIG_NUMA
+static inline int virt_to_nodeid(const void *obj)
+{
+	struct page *page = virt_to_page(obj);
+#ifdef KEEP_NODEID_IN_SLAB
+	return page_get_slab(page)->nodeid;
+#else
+	return (long)page->lru.next & 63;
+#endif
+}
+#endif
+
 static inline struct slab *virt_to_slab(const void *obj)
 {
 	struct page *page = virt_to_page(obj);
@@ -1134,8 +1166,7 @@ static void drain_alien_cache(struct kme
 
 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
 {
-	struct slab *slabp = virt_to_slab(objp);
-	int nodeid = slabp->nodeid;
+	int nodeid = virt_to_nodeid(objp);
 	struct kmem_list3 *l3;
 	struct array_cache *alien = NULL;
 	int node;
@@ -1146,7 +1177,7 @@ static inline int cache_free_alien(struc
 	 * Make sure we are not freeing a object from another node to the array
 	 * cache on this cpu.
 	 */
-	if (likely(slabp->nodeid == node) || unlikely(!use_alien_caches))
+	if (likely(nodeid == node) || unlikely(!use_alien_caches))
 		return 0;
 
 	l3 = cachep->nodelists[node];
@@ -1437,8 +1468,14 @@ void __init kmem_cache_init(void)
 	cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
 	cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE];
 
+#ifdef KEEP_NODEID_IN_PAGE
+	/* kmem_cache addresses must be multiple of 64 */
+	cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
+					max(64, cache_line_size()));
+#else
 	cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
 					cache_line_size());
+#endif
 	cache_cache.reciprocal_buffer_size =
 		reciprocal_value(cache_cache.buffer_size);
 
@@ -2588,7 +2625,6 @@ static struct slab *alloc_slabmgmt(struc
 	slabp->inuse = 0;
 	slabp->colouroff = colour_off;
 	slabp->s_mem = objp + colour_off;
-	slabp->nodeid = nodeid;
 	return slabp;
 }
 
@@ -2699,7 +2735,7 @@ #endif
  * virtual address for kfree, ksize, kmem_ptr_validate, and slab debugging.
  */
 static void slab_map_pages(struct kmem_cache *cache, struct slab *slab,
-			   void *addr)
+			   void *addr, int nodeid)
 {
 	int nr_pages;
 	struct page *page;
@@ -2711,8 +2747,7 @@ static void slab_map_pages(struct kmem_c
 		nr_pages <<= cache->gfporder;
 
 	do {
-		page_set_cache(page, cache);
-		page_set_slab(page, slab);
+		page_set_cache_slab_nodeid(page, cache, slab, nodeid);
 		page++;
 	} while (--nr_pages);
 }
@@ -2787,8 +2822,7 @@ static int cache_grow(struct kmem_cache 
 	if (!slabp)
 		goto opps1;
 
-	slabp->nodeid = nodeid;
-	slab_map_pages(cachep, slabp, objp);
+	slab_map_pages(cachep, slabp, objp, nodeid);
 
 	cache_init_objs(cachep, slabp, ctor_flags);
 
#1 lore raw
Quoted 2 lines · Eric Dumazet

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

It would be nice if you could cc me for slab patches.

mm/slab.c static int slab_break_gfp_order = BREAK_ · line 611

Quoted 13 lines · Eric Dumazet
mm/slab.c +10 -1
-static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
+static inline void page_set_cache_slab_nodeid(struct page *page,
+       struct kmem_cache *cache, struct slab *slab, int nodeid)
 {
+       page->lru.prev = (struct list_head *)slab;
+#ifdef KEEP_NODEID_IN_PAGE
+       page->lru.next = (struct list_head *)((long)cache + nodeid);
+#else
        page->lru.next = (struct list_head *)cache;
+#endif
+#ifdef KEEP_NODEID_IN_SLAB
+       slab->nodeid = nodeid;
+#endif

Can we please have a slab_get_nid() and slab_set_nid() instead which reduces the need for #ifdefs?

#2 lore raw
Quoted 2 lines · Pekka Enberg

Can we please have a slab_get_nid() and slab_set_nid() instead which reduces the need for #ifdefs?

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

#3 lore raw
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Sounds good. Do you know for a fact that we don't or are you just afraid we have a bug?

#4 lore raw
Quoted 7 lines · Pekka Enberg
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Sounds good. Do you know for a fact that we don't or are you just afraid we have a bug?

I tried to clean it up during the GFP_THISNODE work but there was some issue on powerpc where we needed fallback for bootstrap. There may be more trouble there. Checking the allocs where we do not use GFP_THISNODE should get you there.

#5 lore raw
In reply to Christoph Lameter
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Oh well, here's a patch to find out. (I don't have a NUMA box.)

Pekka

From: Pekka Enberg <[email protected]>

This adds a virt_to_nodeid() for looking up the NUMA node ID for a slab object. The current implementation uses slab->nodeid but adds a WARN_ON to catch cases where page nid differs. Eventually, we can change virt_to_nodeid() to use page_to_nid after which slab->nodeid can be removed.

---
 mm/slab.c |   17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)
mm/slab.c +12 -5
Index: 2.6/mm/slab.c
===================================================================
@@ -623,6 +623,14 @@ static inline struct slab *virt_to_slab(
 	return page_get_slab(page);
 }
 
+static inline unsigned short virt_to_nodeid(const void *obj)
+{
+	struct page *page = virt_to_page(obj);
+	struct slab *slab = page_get_slab(page);
+	WARN_ON(slab->nodeid != page_to_nid(page));
+	return slab->nodeid;
+}
+
 static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
 				 unsigned int idx)
 {
@@ -1134,8 +1142,7 @@ 	int i = 0;
 
 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
 {
-	struct slab *slabp = virt_to_slab(objp);
-	int nodeid = slabp->nodeid;
+	int nodeid = virt_to_nodeid(objp);
 	struct kmem_list3 *l3;
 	struct array_cache *alien = NULL;
 	int node;
@@ -1146,7 +1153,7 @@ static inline int cache_free_alien(struc
 	 * Make sure we are not freeing a object from another node to the array
 	 * cache on this cpu.
 	 */
-	if (likely(slabp->nodeid == node) || unlikely(!use_alien_caches))
+	if (likely(nodeid == node) || unlikely(!use_alien_caches))
 		return 0;
 
 	l3 = cachep->nodelists[node];
@@ -2665,7 +2672,7 @@ static void *slab_get_obj(struct kmem_ca
 	next = slab_bufctl(slabp)[slabp->free];
 #if DEBUG
 	slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
-	WARN_ON(slabp->nodeid != nodeid);
+	WARN_ON(virt_to_nodeid(objp) != nodeid);
 #endif
 	slabp->free = next;
 
@@ -2679,7 +2686,7 @@ static void slab_put_obj(struct kmem_cac
 
 #if DEBUG
 	/* Verify that the slab belongs to the intended node */
-	WARN_ON(slabp->nodeid != nodeid);
+	WARN_ON(virt_to_nodeid(objp) != nodeid);
 
 	if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
 		printk(KERN_ERR "slab: double free detected in cache "
#6 lore raw
Quoted 7 lines · On Wed, 21 Mar 2007 17:42:22 +0200 (EET) Pekka J Enberg…
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Oh well, here's a patch to find out. (I don't have a NUMA box.)

This wont work, because of use_alien_caches knob.

If use_alien_caches is set to 0, your WARN_ON will trigger many false alarms

#7 lore raw
Quoted 4 lines · Eric Dumazet

This wont work, because of use_alien_caches knob.

If use_alien_caches is set to 0, your WARN_ON will trigger many false alarms

How is that? The mapping should always be the same regardless of whether we use alien caches or not.

#8 lore raw
In reply to Pekka J Enberg
Quoted 6 lines · Pekka J Enberg
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Oh well, here's a patch to find out. (I don't have a NUMA box.)

You can always use numa emulation on x86-64. Boot with numa=fake=NODES

-Andi

#9 lore raw
Quoted 2 lines · Andi Kleen

You can always use numa emulation on x86-64. Boot with numa=fake=NODES

Yes, I know, but I don't have a x86-64 box either =)

#10 lore raw
In reply to Andi Kleen
Quoted 9 lines · Andi Kleen
Quoted 6 lines · Pekka J Enberg
Quoted 3 lines · Christoph Lameter

None of that please. The page flags already contain a node number that is accessible via page_to_nid(). Just make sure that we never put a slab onto the wrong node.

Oh well, here's a patch to find out. (I don't have a NUMA box.)

You can always use numa emulation on x86-64. Boot with numa=fake=NODES

This wont help you to debug strange fallback scenarios on unusual NUMA platforms.

#11 lore raw

[RFC] NUMA : could we introduce virt_to_nid() ?

In reply to Andi Kleen

Hi Andi

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

/* common implementation *
int virt_to_nid(const void *addr)

{
	struct page *page = virt_to_page(addr); 
	return page_to_nid(page);
}

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

Thank you

#12 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

Quoted 5 lines · Eric Dumazet

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

/* common implementation *
int virt_to_nid(const void *addr)

Quoted 7 lines
{
        struct page *page = virt_to_page(addr);
        return page_to_nid(page);
}

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

Sounds good to me. At least cache_free_alien() in mm/slab.c to should be converted to use it.

#13 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

Quoted 11 lines · On Fri, 23 Mar 2007 14:48:24 +0200 "Pekka Enberg" <[email protected]>…
Quoted 10 lines · Eric Dumazet

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

/* common implementation *
int virt_to_nid(const void *addr)

{
        struct page *page = virt_to_page(addr);
        return page_to_nid(page);
}
Quoted 6 lines
Quoted 2 lines

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

Sounds good to me. At least cache_free_alien() in mm/slab.c to should be converted to use it.

Not yet :(

Because in slab we currently need virt_to_slab(addr)->nodeid, not page_to_nid(virt_to_page(addr))

It might be different, according to Christoph

#14 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

In reply to Eric Dumazet
Quoted 4 lines · Eric Dumazet

Hi Andi

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

Quoted 8 lines

/* common implementation *
int virt_to_nid(const void *addr)

{
	struct page *page = virt_to_page(addr); 
	return page_to_nid(page);
}

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

Sure why not. Please send a patch.

-Andi

#15 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

In reply to Eric Dumazet
Quoted 11 lines · Eric Dumazet

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

/* common implementation *
int virt_to_nid(const void *addr)

{
	struct page *page = virt_to_page(addr); 
	return page_to_nid(page);
}

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

This is going to pollute the caches since we then use multiple ways to determine the node of a page. Its better to stay with the same approach for all pages. The page struct is used for many other purposes as well. Its likely to be in the cpu cache.

#16 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

Quoted 11 lines · On Fri, 23 Mar 2007 07:50:28 -0700 (PDT) Christoph Lameter…
Quoted 9 lines · Eric Dumazet

Checking Christoph quicklist implementation, I found the same cache miss in free() than SLAB has.

/* common implementation *
int virt_to_nid(const void *addr)

{
	struct page *page = virt_to_page(addr); 
	return page_to_nid(page);
}
Quoted 8 lines
Quoted 2 lines

On some platforms (x86_64 for example), could we have a better implementation, not accessing struct page, but using phys_to_nid() ?

This is going to pollute the caches since we then use multiple ways to determine the node of a page. Its better to stay with the same approach for all pages. The page struct is used for many other purposes as well. Its likely to be in the cpu cache.

Sorry ? page structs are not in cpu cache at all.

phys_to_nid() on my 16 GB x86_64 machine uses one single cache line, shared by all pages. This single cache line is cache hot yes, not "struct page"s ...

And on this machine, thats about 224 Mbytes of 'struct pages'

You carefully commented your alloc() function saying it is touching two cache lines. But you omited to say that free() function needs 3 cache lines if CONFIG_NUMA

For SLAB use, page struct is needed because we use lru.{next|prev} to store slab/cachep pointers, but for a pure page allocator, unless I misread your patch, we dont need it, if virt_to_nid() can do its job without it.

#17 lore raw

Re: [RFC] NUMA : could we introduce virt_to_nid() ?

Quoted 1 line · Eric Dumazet

Sorry ? page structs are not in cpu cache at all.

They are if they are in any way handled by the VM.

Quoted 2 lines

You carefully commented your alloc() function saying it is touching two cache lines. But you omited to say that free() function needs 3 cache lines if CONFIG_NUMA

Yes it needs the third cacheline to check the node of the page in the NUMA case.

Quoted 4 lines

For SLAB use, page struct is needed because we use lru.{next|prev} to store slab/cachep pointers, but for a pure page allocator, unless I misread your patch, we dont need it, if virt_to_nid() can do its job without it.

The page allocator is only handling page_structs.

#18 lore raw
In reply to Eric Dumazet

mm/slab.c void __init kmem_cache_init(void) · line 1478

Quoted 8 lines · Eric Dumazet
mm/slab.c +6 -0
+#ifdef KEEP_NODEID_IN_PAGE
+       /* kmem_cache addresses must be multiple of 64 */
+       cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
+                                       max(64, cache_line_size()));
+#else
        cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
                                        cache_line_size());
+#endif

Please introduce cache_cache_align() to hide the #ifdeffing.

#19 lore raw
In reply to Eric Dumazet
Quoted 3 lines · Eric Dumazet

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

1) MAX_NUMNODES <= 64

MAX_NUMNODES == 1024 on IA64 and may be on x86_64 in the future.

Quoted 5 lines

2) alignment of 'struct kmem_cache *' can be >= 64

The following patch changes the page->lru.next to contain not only the 'struct kmem_cache *' pointer, but also the nodeid in the low order bits.

Nack.

I would suggest you audit slab and make sure that we never fall back and always enter a slab on the node of page_to_nid(page). If
all allocations use GFP_THISNODE then you should be safe and then we can remove the nodeid from the slab structure and simply use the available node information in the page struct.

#20 lore raw
Quoted 7 lines · On Wed, 21 Mar 2007 07:41:46 -0700 (PDT) Christoph Lameter…
Quoted 3 lines · Eric Dumazet

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

1) MAX_NUMNODES <= 64

MAX_NUMNODES == 1024 on IA64 and may be on x86_64 in the future.

So what ? I am very happy for you Christoph.

Average linux machines (ie more than 99.99 % on this planet) are using MAX_NUMNODES <= 64

If you read my patch, your lovely 1024 nodes machines will run as before : No change at all for big machines.

Quoted 7 lines
Quoted 6 lines

2) alignment of 'struct kmem_cache *' can be >= 64

The following patch changes the page->lru.next to contain not only the 'struct kmem_cache *' pointer, but also the nodeid in the low order bits.

Quoted 8 lines

Nack.

I would suggest you audit slab and make sure that we never fall back and always enter a slab on the node of page_to_nid(page). If
all allocations use GFP_THISNODE then you should be safe and then we can remove the nodeid from the slab structure and simply use the available node information in the page struct.

Last time I checked 'struct page', they was no nodeid in it.

I cooked this patch because you told me it was possible to have a 'slab nodeid' different than page_to_nid(obj). Now you want me to prove the contrary. Me confused.

#21 lore raw
Quoted 5 lines · Eric Dumazet
Quoted 2 lines · On Wed, 21 Mar 2007 07:41:46 -0700 (PDT) Christoph Lameter…

On Wed, 21 Mar 2007, Eric Dumazet wrote:

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

1) MAX_NUMNODES <= 64

MAX_NUMNODES == 1024 on IA64 and may be on x86_64 in the future.

So what ? I am very happy for you Christoph.

You wanted to exploit that MAX_NUMNODES <= 64?

#22 lore raw
Quoted 1 line · On Wed, 21 Mar 2007 08:44:53 -0700 (PDT) Christoph Lameter…

You wanted to exploit that MAX_NUMNODES <= 64?

Maybe my english is not perfect. I promise I improve it eventually.

My patch was for unlucky guys that have MAX_NUMNODES <= 64.

For others, the preprocessor gave the old code.

#23 lore raw
In reply to Eric Dumazet
Quoted 1 line · Eric Dumazet

Last time I checked 'struct page', they was no nodeid in it.

Hmm, page_to_nid() in include/linx/mm.h doesn't agree with you:

#ifdef NODE_NOT_IN_PAGE_FLAGS
extern int page_to_nid(struct page *page);
#else
static inline int page_to_nid(struct page *page)

{
       return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
}

#endif

#24 lore raw
Quoted 5 lines · On Wed, 21 Mar 2007 17:47:13 +0200 "Pekka Enberg" <[email protected]>…
Quoted 1 line · Eric Dumazet

Last time I checked 'struct page', they was no nodeid in it.

Hmm, page_to_nid() in include/linx/mm.h doesn't agree with you:

Quoted 9 lines

#ifdef NODE_NOT_IN_PAGE_FLAGS
extern int page_to_nid(struct page *page);
#else
static inline int page_to_nid(struct page *page)

{
        return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
}

#endif

Correct, but still another ifdef NODE_NOT_IN_PAGE_FLAGS

Anyway the correct path is to check GFP_THISNODE stuff, because we can even avoid access to 'struct page' to get the nodeid, at least on x86_64 NUMA, where struct memnode can provide the information with less cache footprint.

#25 lore raw
In reply to Eric Dumazet
Quoted 8 lines · Eric Dumazet

In order to avoid a cache miss in kmem_cache_free() on NUMA and reduce hot path length, we could exploit the following common facts.

1) MAX_NUMNODES <= 64

2) alignment of 'struct kmem_cache *' can be >= 64

The following patch changes the page->lru.next to contain not only the 'struct kmem_cache *' pointer, but also the nodeid in the low order bits.

It might be cleaner to just put it into flags on 64bit. There is enough space there and we don't really care about 32bit NUMA machines.

-Andi