make slab gfp fair

include/linux/slub_def.h +12 -2

@@ -1371,8 +1376,12 @@ static void *__slab_alloc(struct kmem_ca
 		gfp_t gfpflags, int node, void *addr, struct page *page)
 {
 	void **object;
-	int cpu = smp_processor_id();
+	int cpu;
+
+	if (page == FORCE_PAGE)
+		goto force_new;
 
+	cpu = smp_processor_id();
 	if (!page)
 		goto new_slab;
 

@@ -1405,6 +1414,7 @@ have_slab:
 		goto load_freelist;
 	}
 
+force_new:
 	page = new_slab(s, gfpflags, node);
 	if (page) {
 		cpu = smp_processor_id();

@@ -1465,15 +1475,22 @@ static void __always_inline *slab_alloc(
 	struct page *page;
 	void **object;
 	unsigned long flags;
+	int rank = slab_alloc_rank(gfpflags);
 
 	local_irq_save(flags);
+	if (slab_insufficient_rank(s, rank)) {
+		page = FORCE_PAGE;
+		goto force_alloc;
+	}
+
 	page = s->cpu_slab[smp_processor_id()];
 	if (unlikely(!page || !page->lockless_freelist ||
-			(node != -1 && page_to_nid(page) != node)))

include/linux/slub_def.h struct kmem_cache { · line 56

Quoted 10 lines · Peter Zijlstra

Index: linux-2.6-git/include/linux/slub_def.h
===================================================================

include/linux/slub_def.h +1 -0

@@ -52,6 +52,7 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
 	struct page *cpu_slab[NR_CPUS];
+	int rank;
 };

Ranks as part of the kmem_cache structure? I thought this is a temporary thing?

include/linux/slub_def.h static struct page *allocate_slab(struct · line 966

Quoted 7 lines

include/linux/slub_def.h +2 -0

  * Lock order:

@@ -961,6 +962,8 @@ static struct page *allocate_slab(struct
 	if (!page)
 		return NULL;
 
+	s->rank = page->index;
+

Argh.... Setting a cache structure field from a page struct field? What about concurrency?

Quoted 42 lines

include/linux/slub_def.h +14 -2

@@ -1371,8 +1376,12 @@ static void *__slab_alloc(struct kmem_ca
 		gfp_t gfpflags, int node, void *addr, struct page *page)
 {
 	void **object;
-	int cpu = smp_processor_id();
+	int cpu;
+
+	if (page == FORCE_PAGE)
+		goto force_new;
 
+	cpu = smp_processor_id();
 	if (!page)
 		goto new_slab;
 

@@ -1405,6 +1414,7 @@ have_slab:
 		goto load_freelist;
 	}
 
+force_new:
 	page = new_slab(s, gfpflags, node);
 	if (page) {
 		cpu = smp_processor_id();

@@ -1465,15 +1475,22 @@ static void __always_inline *slab_alloc(
 	struct page *page;
 	void **object;
 	unsigned long flags;
+	int rank = slab_alloc_rank(gfpflags);
 
 	local_irq_save(flags);
+	if (slab_insufficient_rank(s, rank)) {
+		page = FORCE_PAGE;
+		goto force_alloc;
+	}
+
 	page = s->cpu_slab[smp_processor_id()];
 	if (unlikely(!page || !page->lockless_freelist ||
-			(node != -1 && page_to_nid(page) != node)))
+			(node != -1 && page_to_nid(page) != node))) {
 
+force_alloc:
 		object = __slab_alloc(s, gfpflags, node, addr, page);
 

include/linux/slub_def.h static void __always_inline *slab_alloc( · line 1496

Quoted 5 lines

include/linux/slub_def.h +1 -1

-	else {
+	} else {
 		object = page->lockless_freelist;
 		page->lockless_freelist = object[page->offset];
 	}

This is the hot path. No modifications please.

Quoted 15 lines · Christoph Lameter

include/linux/slub_def.h struct kmem_cache { · line 56

Quoted 10 lines · Peter Zijlstra

Index: linux-2.6-git/include/linux/slub_def.h
===================================================================

include/linux/slub_def.h +1 -0

@@ -52,6 +52,7 @@ struct kmem_cache {
 	struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
 	struct page *cpu_slab[NR_CPUS];
+	int rank;
 };

Ranks as part of the kmem_cache structure? I thought this is a temporary thing?

No it needs to store the current state to verity subsequent allocations their gfp flags against.

Quoted 5 lines

Quoted 5 lines

include/linux/slub_def.h +0 -0

  * Lock order:

@@ -961,6 +962,8 @@ static struct page *allocate_slab(struct
 	if (!page)
 		return NULL;
 

Quoted 5 lines

include/linux/slub_def.h static struct page *allocate_slab(struct · line 966

Quoted 2 lines

include/linux/slub_def.h +2 -0

+	s->rank = page->index;
+

Argh.... Setting a cache structure field from a page struct field? What about concurrency?

Oh, right; allocate_slab is not serialized itself.

Quoted 8 lines

Quoted 6 lines

include/linux/slub_def.h +1 -1

-	else {
+	} else {
 		object = page->lockless_freelist;
 		page->lockless_freelist = object[page->offset];

}

This is the hot path. No modifications please.

Yes it is, but sorry, I have to. I really need to validate each slab alloc its GFP flags. Thats what the whole thing is about, I thought you understood that.

In the interest of creating a reserve based allocator; we need to make the slab allocator (*sigh*, all three) fair with respect to GFP flags.

That is, we need to protect memory from being used by easier gfp flags than it was allocated with. If our reserve is placed below GFP_ATOMIC, we do not want a GFP_KERNEL allocation to walk away with it - a scenario that is perfectly possible with the current allocators.

Quoted 2 lines · Peter Zijlstra

In the interest of creating a reserve based allocator; we need to make the slab allocator (*sigh*, all three) fair with respect to GFP flags.

I am not sure what the point of all of this is.

Quoted 4 lines

That is, we need to protect memory from being used by easier gfp flags than it was allocated with. If our reserve is placed below GFP_ATOMIC, we do not want a GFP_KERNEL allocation to walk away with it - a scenario that is perfectly possible with the current allocators.

Why does this have to handled by the slab allocators at all? If you have free pages in the page allocator then the slab allocators will be able to use that reserve.

Quoted 3 lines

Why does this have to handled by the slab allocators at all? If you have free pages in the page allocator then the slab allocators will be able to use that reserve.

Yes, too freely. GFP flags are only ever checked when you allocate a new page. Hence, if you have a low reaching alloc allocating a slab page; subsequent non critical GFP_KERNEL allocs can fill up that slab. Hence you would need to reserve a slab per object instead of the normal packing.

Quoted 2 lines · Peter Zijlstra

In the interest of creating a reserve based allocator; we need to make the slab allocator (*sigh*, all three) fair with respect to GFP flags.

I am not sure what the point of all of this is.

Quoted 4 lines

That is, we need to protect memory from being used by easier gfp flags than it was allocated with. If our reserve is placed below GFP_ATOMIC, we do not want a GFP_KERNEL allocation to walk away with it - a scenario that is perfectly possible with the current allocators.

Why does this have to handled by the slab allocators at all? If you have free pages in the page allocator then the slab allocators will be able to use that reserve.

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool

do_io()

                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       ...
                                          eats it all

kmem_cache_alloc(...) -> ENOMEM
who ate my donuts?!

But I think this solution is somehow overkill. If we only care about this issue in the OOM avoidance case, then our rank reduces to a boolean.

Signature · 1 line

Mathematics is the supreme nostalgia of our time.

Quoted 3 lines · Matt Mackall

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool

Yes but it returns an object for the privileged thread. Is that not enough?

Quoted 11 lines

do_io()

                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       ...
                                          eats it all

kmem_cache_alloc(...) -> ENOMEM
who ate my donuts?!

Quoted 3 lines

But I think this solution is somehow overkill. If we only care about this issue in the OOM avoidance case, then our rank reduces to a boolean.

Quoted 6 lines · Christoph Lameter

Quoted 3 lines · Matt Mackall

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool

Quoted 2 lines

Yes but it returns an object for the privileged thread. Is that not enough?

No, because we reserved memory for n objects, and like matt illustrates most of those that will be eaten by the greedy process.
We could reserve 1 page per object but that rather bloats the reserve.

Quoted 7 lines · Peter Zijlstra

Quoted 6 lines · Christoph Lameter

Quoted 3 lines · Matt Mackall

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool

Quoted 6 lines

Quoted 2 lines

Yes but it returns an object for the privileged thread. Is that not enough?

No, because we reserved memory for n objects, and like matt illustrates most of those that will be eaten by the greedy process.
We could reserve 1 page per object but that rather bloats the reserve.

1 slab per object not one page. But yes thats some bloat.

You can pull the big switch (only on a SLUB slab I fear) to switch off the fast path. Do SetSlabDebug() when allocating a precious allocation that should not be gobbled up by lower level processes. Then you can do whatever you want in the __slab_alloc debug section and we wont care because its not the hot path.

SLAB is a bit different. There we already have issues with the fast path due to the attempt to handle numa policies at the object level. SLUB fixes that issue (if we can avoid you hot path patch). It intentionally does defer all special object handling to the slab level to increase NUMA performance. If you do the same to SLAB then you will get the NUMA troubles propagated to the SMP and UP level.

Quoted 5 lines

You can pull the big switch (only on a SLUB slab I fear) to switch off the fast path. Do SetSlabDebug() when allocating a precious allocation that should not be gobbled up by lower level processes. Then you can do whatever you want in the __slab_alloc debug section and we wont care because its not the hot path.

One allocator is all I need; it would just be grand if all could be supported.

So what you suggest is not placing the 'emergency' slab into the regular place so that normal allocations will not be able to find it. Then if an emergency allocation cannot be satified by the regular path, we fall back to the slow path and find the emergency slab.

The thing is; I'm not needing any speed, as long as the machine stay alive I'm good. However others are planing to build a full reserve based allocator to properly fix the places that now use __GFP_NOFAIL and situation such as in add_to_swap().

Quoted 6 lines · Peter Zijlstra

Quoted 5 lines

You can pull the big switch (only on a SLUB slab I fear) to switch off the fast path. Do SetSlabDebug() when allocating a precious allocation that should not be gobbled up by lower level processes. Then you can do whatever you want in the __slab_alloc debug section and we wont care because its not the hot path.

Quoted 7 lines

One allocator is all I need; it would just be grand if all could be supported.

So what you suggest is not placing the 'emergency' slab into the regular place so that normal allocations will not be able to find it. Then if an emergency allocation cannot be satified by the regular path, we fall back to the slow path and find the emergency slab.

Hmmm.. Maybe we could do that.... But what I had in mind was simply to set a page flag (DebugSlab()) if you know in alloc_slab that the slab should be only used for emergency allocation. If DebugSlab is set then the fastpath will not be called. You can trap all allocation attempts and insert whatever fancy logic you want in the debug path since its not performance critical.

Quoted 4 lines

The thing is; I'm not needing any speed, as long as the machine stay alive I'm good. However others are planing to build a full reserve based allocator to properly fix the places that now use __GFP_NOFAIL and situation such as in add_to_swap().

Well I have version of SLUB here that allows you do redirect the alloc calls at will. Adds a kmem_cache_ops structure and in the kmem_cache_ops structure you can redirect allocation and freeing of slabs (not objects!) at will. Would that help?

Quoted 6 lines

Hmmm.. Maybe we could do that.... But what I had in mind was simply to set a page flag (DebugSlab()) if you know in alloc_slab that the slab should be only used for emergency allocation. If DebugSlab is set then the fastpath will not be called. You can trap all allocation attempts and insert whatever fancy logic you want in the debug path since its not performance critical.

I might have missed some detail when I looked at SLUB, but I did not see how setting SlabDebug would trap subsequent allocations to that slab.

Quoted 7 lines · Peter Zijlstra

Quoted 6 lines

Hmmm.. Maybe we could do that.... But what I had in mind was simply to set a page flag (DebugSlab()) if you know in alloc_slab that the slab should be only used for emergency allocation. If DebugSlab is set then the fastpath will not be called. You can trap all allocation attempts and insert whatever fancy logic you want in the debug path since its not performance critical.

Quoted 2 lines

I might have missed some detail when I looked at SLUB, but I did not see how setting SlabDebug would trap subsequent allocations to that slab.

Ok its not evident in slab_alloc. But if SlabDebug is set then page->lockless_list is always NULL and we always fall back to __slab_alloc.

There we check for SlabDebug and go to the debug: label.
There you can insert any fancy processing you want.

Quoted 9 lines

Quoted 4 lines

The thing is; I'm not needing any speed, as long as the machine stay alive I'm good. However others are planing to build a full reserve based allocator to properly fix the places that now use __GFP_NOFAIL and situation such as in add_to_swap().

Well I have version of SLUB here that allows you do redirect the alloc calls at will. Adds a kmem_cache_ops structure and in the kmem_cache_ops structure you can redirect allocation and freeing of slabs (not objects!) at will. Would that help?

I'm not sure; I need kmalloc as well.

One allocator is all I need; it would just be grand if all could be supported.

So what you suggest is not placing the 'emergency' slab into the regular place so that normal allocations will not be able to find it. Then if an emergency allocation cannot be satified by the regular path, we fall back to the slow path and find the emergency slab.

How about something like this; it seems to sustain a little stress.

Quoted 1 line · Peter Zijlstra

How about something like this; it seems to sustain a little stress.

Argh again mods to kmem_cache.

Quoted 5 lines · Christoph Lameter

Quoted 1 line · Peter Zijlstra

How about something like this; it seems to sustain a little stress.

Argh again mods to kmem_cache.

Hmm, I had not understood you minded that very much; I did stay away from all the fast paths this time.

The thing is, I wanted to fold all the emergency allocs into a single slab, not a per cpu thing. And once you loose the per cpu thing, you need some extra serialization. Currently the top level lock is slab_lock(page), but that only works because we have interrupts disabled and work per cpu.

Why is it bad to extend kmem_cache a bit?

Quoted 9 lines · Peter Zijlstra

Quoted 5 lines · Christoph Lameter

Quoted 1 line · Peter Zijlstra

How about something like this; it seems to sustain a little stress.

Argh again mods to kmem_cache.

Hmm, I had not understood you minded that very much; I did stay away from all the fast paths this time.

Well you added a new locking level and changed the locking hierachy!

Quoted 5 lines

The thing is, I wanted to fold all the emergency allocs into a single slab, not a per cpu thing. And once you loose the per cpu thing, you need some extra serialization. Currently the top level lock is slab_lock(page), but that only works because we have interrupts disabled and work per cpu.

SLUB can only allocate from a per cpu slab. You will have to reserve one slab per cpu anyways unless we flush the cpu slab after each access. Same thing is true for SLAB. It wants objects in its per cpu queues.

Quoted 1 line

Why is it bad to extend kmem_cache a bit?

Because it is for all practical purposes a heavily accessed read only structure. Modifications only occur to per node and per cpu structures. In a 4k systems any write will kick out the kmem_cache cacheline in 4k processors.

If this 4k cpu system ever gets to touch the new lock it is in way deeper problems than a bouncing cache-line.

Please look at it more carefully.

We differentiate pages allocated at the level where GFP_ATOMIC starts to fail. By not updating the percpu slabs those are retried every time, except for ALLOC_NO_WATERMARKS allocations; those are served from the ->reserve_slab.

Once a regular slab allocation succeeds again, the ->reserve_slab is cleaned up and never again looked at it until we're in distress again.

Signed-off-by Peter Zijlstra <[email protected]>

---
 include/linux/slub_def.h |    2 +
 mm/slub.c                |   85 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 78 insertions(+), 9 deletions(-)

Index: linux-2.6-git/include/linux/slub_def.h
===================================================================

include/linux/slub_def.h +2 -0

@@ -46,6 +46,8 @@ struct kmem_cache {
 	struct list_head list;	/* List of slab caches */
 	struct kobject kobj;	/* For sysfs */
 
+	struct page *reserve_slab;
+
 #ifdef CONFIG_NUMA
 	int defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];

Index: linux-2.6-git/mm/slub.c
===================================================================

mm/slub.c +9 -4

@@ -20,11 +20,13 @@
 #include <linux/mempolicy.h>
 #include <linux/ctype.h>
 #include <linux/kallsyms.h>
+#include "internal.h"
 
 /*
  * Lock order:
- *   1. slab_lock(page)
- *   2. slab->list_lock
+ *   1. reserve_lock
+ *   2. slab_lock(page)
+ *   3. node->list_lock
  *
  *   The slab_lock protects operations on the object of a particular
  *   slab and its metadata in the page struct. If the slab lock

@@ -259,6 +261,8 @@ static int sysfs_slab_alias(struct kmem_
 static void sysfs_slab_remove(struct kmem_cache *s) {}
 #endif
 
+static DEFINE_SPINLOCK(reserve_lock);
+
 /********************************************************************
  * 			Core slab cache functions
  *******************************************************************/

@@ -1007,7 +1011,7 @@ static void setup_object(struct kmem_cac
 		s->ctor(object, s, 0);
 }
 
-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node, int *rank)
 {
 	struct page *page;
 	struct kmem_cache_node *n;

@@ -1025,6 +1029,7 @@ static struct page *new_slab(struct kmem
 	if (!page)
 		goto out;
 
+	*rank = page->rank;
 	n = get_node(s, page_to_nid(page));
 	if (n)
 		atomic_long_inc(&n->nr_slabs);

@@ -1311,7 +1316,7 @@ static void unfreeze_slab(struct kmem_ca
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
+static void __deactivate_slab(struct kmem_cache *s, struct page *page)
 {
 	/*
 	 * Merge cpu freelist into freelist. Typically we get here

mm/slub.c +1 -0

@@ -1395,6 +1405,7 @@ static void *__slab_alloc(struct kmem_ca
 {
 	void **object;
 	int cpu = smp_processor_id();
+	int rank = 0;
 
 	if (!page)
 		goto new_slab;

mm/slub.c +18 -0

+			}
+		}
 		cpu = smp_processor_id();
 		if (s->cpu_slab[cpu]) {
 			/*

@@ -1455,6 +1482,18 @@ new_slab:
 		SetSlabFrozen(page);
 		s->cpu_slab[cpu] = page;
 		goto load_freelist;
+	} else if (page) {
+		spin_lock(&reserve_lock);
+		if (s->reserve_slab) {
+			discard_slab(s, page);
+			page = s->reserve_slab;
+		}
+		slab_lock(page);
+		SetPageActive(page);
+		s->reserve_slab = page;
+		spin_unlock(&reserve_lock);
+
+		goto got_reserve;
 	}
 	return NULL;
 debug:

@@ -1470,6 +1509,31 @@ debug:
 	page->freelist = object[page->offset];
 	slab_unlock(page);
 	return object;
+
+try_reserve:
+	spin_lock(&reserve_lock);
+	page = s->reserve_slab;

Quoted 2 lines · Peter Zijlstra

If this 4k cpu system ever gets to touch the new lock it is in way deeper problems than a bouncing cache-line.

So its no use on NUMA?

Quoted 7 lines

Please look at it more carefully.

We differentiate pages allocated at the level where GFP_ATOMIC starts to fail. By not updating the percpu slabs those are retried every time, except for ALLOC_NO_WATERMARKS allocations; those are served from the ->reserve_slab.

Quoted 2 lines

Once a regular slab allocation succeeds again, the ->reserve_slab is cleaned up and never again looked at it until we're in distress again.

A single slab? This may only give you a a single object in an extreme case. Are you sure that this solution is generic enough?

The problem here is that you may spinlock and take out the slab for one cpu but then (AFAICT) other cpus can still not get their high priority allocs satisfied. Some comments follow.

Quoted 77 lines

Signed-off-by Peter Zijlstra <[email protected]>

---
 include/linux/slub_def.h |    2 +
 mm/slub.c                |   85 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 78 insertions(+), 9 deletions(-)

Index: linux-2.6-git/include/linux/slub_def.h
===================================================================

include/linux/slub_def.h +2 -0

@@ -46,6 +46,8 @@ struct kmem_cache {
 	struct list_head list;	/* List of slab caches */
 	struct kobject kobj;	/* For sysfs */
 
+	struct page *reserve_slab;
+
 #ifdef CONFIG_NUMA
 	int defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];

Index: linux-2.6-git/mm/slub.c
===================================================================

mm/slub.c +9 -4

@@ -20,11 +20,13 @@
 #include <linux/mempolicy.h>
 #include <linux/ctype.h>
 #include <linux/kallsyms.h>
+#include "internal.h"
 
 /*
  * Lock order:
- *   1. slab_lock(page)
- *   2. slab->list_lock
+ *   1. reserve_lock
+ *   2. slab_lock(page)
+ *   3. node->list_lock
  *
  *   The slab_lock protects operations on the object of a particular
  *   slab and its metadata in the page struct. If the slab lock

@@ -259,6 +261,8 @@ static int sysfs_slab_alias(struct kmem_
 static void sysfs_slab_remove(struct kmem_cache *s) {}
 #endif
 
+static DEFINE_SPINLOCK(reserve_lock);
+
 /********************************************************************
  * 			Core slab cache functions
  *******************************************************************/

@@ -1007,7 +1011,7 @@ static void setup_object(struct kmem_cac
 		s->ctor(object, s, 0);
 }
 
-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node, int *rank)
 {
 	struct page *page;
 	struct kmem_cache_node *n;

@@ -1025,6 +1029,7 @@ static struct page *new_slab(struct kmem
 	if (!page)
 		goto out;
 
+	*rank = page->rank;
 	n = get_node(s, page_to_nid(page));
 	if (n)
 		atomic_long_inc(&n->nr_slabs);

@@ -1311,7 +1316,7 @@ static void unfreeze_slab(struct kmem_ca
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
+static void __deactivate_slab(struct kmem_cache *s, struct page *page)
 {
 	/*
 	 * Merge cpu freelist into freelist. Typically we get here

@@ -1330,10 +1335,15 @@ static void deactivate_slab(struct kmem_
 		page->freelist = object;
 		page->inuse--;

mm/slub.c static void deactivate_slab(struct kmem_ · line 1339

Quoted 4 lines

mm/slub.c +0 -1

 	}
-	s->cpu_slab[cpu] = NULL;
 	unfreeze_slab(s, page);
 }

So you want to spill back the lockless_freelist without deactivating the slab? Why are you using the lockless_freelist at all? If you do not use it then you can call unfreeze_slab. No need for this split.

Quoted 6 lines

mm/slub.c +1 -0

@@ -1395,6 +1405,7 @@ static void *__slab_alloc(struct kmem_ca
 {
 	void **object;
 	int cpu = smp_processor_id();
+	int rank = 0;
 

mm/slub.c new_slab: · line 1439

Quoted 9 lines

mm/slub.c +2 -1

 	if (!page)
 		goto new_slab;

@@ -1424,10 +1435,26 @@ new_slab:
 	if (page) {
 		s->cpu_slab[cpu] = page;
 		goto load_freelist;
-	}
+	} else if (unlikely(gfp_to_alloc_flags(gfpflags) & ALLOC_NO_WATERMARKS))
+		goto try_reserve;

Ok so we are trying to allocate a slab and do not get one thus -> try_reserve.

But this is only working if we are using the slab after explicitly flushing the cpuslabs. Otherwise the slab may be full and we get to alloc_slab.

mm/slub.c new_slab: · line 1443

Quoted 7 lines

mm/slub.c +3 -2

-	page = new_slab(s, gfpflags, node);
-	if (page) {
 
+alloc_slab:
+	page = new_slab(s, gfpflags, node, &rank);
+	if (page && rank) {

Huh? You mean !page?

Quoted 7 lines

mm/slub.c +7 -0

+		if (unlikely(s->reserve_slab)) {
+			struct page *reserve;
+
+			spin_lock(&reserve_lock);
+			reserve = s->reserve_slab;
+			s->reserve_slab = NULL;
+			spin_unlock(&reserve_lock);

mm/slub.c new_slab: · line 1455

Quoted 5 lines

mm/slub.c +5 -0

+
+			if (reserve) {
+				slab_lock(reserve);
+				__deactivate_slab(s, reserve);
+				putback_slab(s, reserve);

Remove the above two lines (they are wrong regardless) and simply make this the cpu slab.

Quoted 13 lines

mm/slub.c +6 -0

+			}
+		}
 		cpu = smp_processor_id();
 		if (s->cpu_slab[cpu]) {
 			/*

@@ -1455,6 +1482,18 @@ new_slab:
 		SetSlabFrozen(page);
 		s->cpu_slab[cpu] = page;
 		goto load_freelist;
+	} else if (page) {
+		spin_lock(&reserve_lock);
+		if (s->reserve_slab) {
+			discard_slab(s, page);

mm/slub.c new_slab: · line 1496

Quoted 8 lines

mm/slub.c +8 -0

+			page = s->reserve_slab;
+		}
+		slab_lock(page);
+		SetPageActive(page);
+		s->reserve_slab = page;
+		spin_unlock(&reserve_lock);
+
+		goto got_reserve;

Quoted 11 lines

mm/slub.c +4 -0

 	}
 	return NULL;
 debug:

@@ -1470,6 +1509,31 @@ debug:
 	page->freelist = object[page->offset];
 	slab_unlock(page);
 	return object;
+
+try_reserve:
+	spin_lock(&reserve_lock);
+	page = s->reserve_slab;

mm/slub.c debug: · line 1525

Quoted 10 lines

mm/slub.c +10 -0

+	if (!page) {
+		spin_unlock(&reserve_lock);
+		goto alloc_slab;
+	}
+
+	slab_lock(page);
+	if (!page->freelist) {
+		s->reserve_slab = NULL;
+		spin_unlock(&reserve_lock);
+		__deactivate_slab(s, page);

replace with unfreeze slab.

mm/slub.c debug: · line 1526

Quoted 1 line

mm/slub.c +1 -0

+		putback_slab(s, page);

Putting back the slab twice.

mm/slub.c debug: · line 1536

Quoted 10 lines

mm/slub.c +10 -0

+		goto alloc_slab;
+	}
+	spin_unlock(&reserve_lock);
+
+got_reserve:
+	object = page->freelist;
+	page->inuse++;
+	page->freelist = object[page->offset];
+	slab_unlock(page);
+	return object;

Quoted 1 line

So its no use on NUMA?

It is, its just that we're swapping very heavily at that point, a bouncing cache-line will not significantly slow down the box compared to waiting for block IO, will it?

Quoted 3 lines

The problem here is that you may spinlock and take out the slab for one cpu but then (AFAICT) other cpus can still not get their high priority allocs satisfied. Some comments follow.

All cpus are redirected to ->reserve_slab when the regular allocations start to fail.

Quoted 3 lines

But this is only working if we are using the slab after explicitly flushing the cpuslabs. Otherwise the slab may be full and we get to alloc_slab.

/me fails to parse.

Quoted 2 lines

Remove the above two lines (they are wrong regardless) and simply make this the cpu slab.

It need not be the same node; the reserve_slab is node agnostic. So here the free page watermarks are good again, and we can forget all about the ->reserve_slab. We just push it on the free/partial lists and forget about it.

But like you said above: unfreeze_slab() should be good, since I don't use the lockless_freelist.

Quoted 5 lines · Peter Zijlstra

Quoted 1 line

So its no use on NUMA?

It is, its just that we're swapping very heavily at that point, a bouncing cache-line will not significantly slow down the box compared to waiting for block IO, will it?

How does all of this interact with

1. cpusets

2. dma allocations and highmem?

3. Containers?

Quoted 6 lines

Quoted 3 lines

The problem here is that you may spinlock and take out the slab for one cpu but then (AFAICT) other cpus can still not get their high priority allocs satisfied. Some comments follow.

All cpus are redirected to ->reserve_slab when the regular allocations start to fail.

And the reserve slab is refilled from page allocator reserves if needed?

Quoted 5 lines

Quoted 3 lines

But this is only working if we are using the slab after explicitly flushing the cpuslabs. Otherwise the slab may be full and we get to alloc_slab.

/me fails to parse.

s->cpu[cpu] is only NULL if the cpu slab was flushed. This is a pretty rare case likely not worth checking.

Quoted 8 lines

Quoted 2 lines

Remove the above two lines (they are wrong regardless) and simply make this the cpu slab.

It need not be the same node; the reserve_slab is node agnostic. So here the free page watermarks are good again, and we can forget all about the ->reserve_slab. We just push it on the free/partial lists and forget about it.

Quoted 2 lines

But like you said above: unfreeze_slab() should be good, since I don't use the lockless_freelist.

You could completely bypass the regular allocation functions and do

object = s->reserve_slab->freelist; s->reserve_slab->freelist = object[s->reserve_slab->offset];

Quoted 7 lines

How does all of this interact with

1. cpusets

2. dma allocations and highmem?

3. Containers?

Much like the normal kmem_cache would do; I'm not changing any of the page allocation semantics.

But its wanted to try the normal cpu_slab path first to detect that the situation has subsided and we can resume normal operation.

Quoted 8 lines · Peter Zijlstra

Quoted 7 lines

How does all of this interact with

1. cpusets

2. dma allocations and highmem?

3. Containers?

Quoted 2 lines

Much like the normal kmem_cache would do; I'm not changing any of the page allocation semantics.

So if we run out of memory on a cpuset then network I/O will still fail?

I do not see any distinction between DMA and regular memory. If we need DMA memory to complete the transaction then this wont work?

Quoted 2 lines

But its wanted to try the normal cpu_slab path first to detect that the situation has subsided and we can resume normal operation.

Is there some indicator somewhere that indicates that we are in trouble? I just see the ranks.

Quoted 2 lines

I do not see any distinction between DMA and regular memory. If we need DMA memory to complete the transaction then this wont work?

If network relies on slabs that are cpuset constrained and the page allocator reserves do not match that, then yes, it goes bang.

Quoted 2 lines

Is there some indicator somewhere that indicates that we are in trouble? I just see the ranks.

Yes, and page->rank will only ever be 0 if the page was allocated with ALLOC_NO_WATERMARKS, and that only ever happens if we're in dire straights and entitled to it.

Otherwise it'll be ALLOC_WMARK_MIN or somesuch.

Quoted 5 lines · Peter Zijlstra

Quoted 2 lines

I do not see any distinction between DMA and regular memory. If we need DMA memory to complete the transaction then this wont work?

If network relies on slabs that are cpuset constrained and the page allocator reserves do not match that, then yes, it goes bang.

So if I put a 32 bit network card in a 64 bit system -> bang?

Quoted 7 lines

Quoted 2 lines

Is there some indicator somewhere that indicates that we are in trouble? I just see the ranks.

Yes, and page->rank will only ever be 0 if the page was allocated with ALLOC_NO_WATERMARKS, and that only ever happens if we're in dire straights and entitled to it.

Quoted 1 line

Otherwise it'll be ALLOC_WMARK_MIN or somesuch.

How we know that we are out of trouble? Just try another alloc and see? If that is the case then we may be failing allocations after the memory situation has cleared up.

Quoted 3 lines

How we know that we are out of trouble? Just try another alloc and see? If that is the case then we may be failing allocations after the memory situation has cleared up.

No, no, for each regular allocation we retry to populate ->cpu_slab with a new slab. If that works we're out of the woods and the ->reserve_slab is cleaned up.

Quoted 6 lines · Peter Zijlstra

Quoted 3 lines

How we know that we are out of trouble? Just try another alloc and see? If that is the case then we may be failing allocations after the memory situation has cleared up.

No, no, for each regular allocation we retry to populate ->cpu_slab with a new slab. If that works we're out of the woods and the ->reserve_slab is cleaned up.

Hmmm.. so we could simplify the scheme by storing the last rank somewheres.

If the alloc has less priority and we can extend the slab then clear up the situation.

If we cannot extend the slab then the alloc must fail.

Could you put the rank into the page flags? On 64 bit at least there should be enough space.

Quoted 2 lines

Hmmm.. so we could simplify the scheme by storing the last rank somewheres.

Not sure how that would help..

Quoted 4 lines · Peter Zijlstra

Quoted 2 lines

Hmmm.. so we could simplify the scheme by storing the last rank somewheres.

Not sure how that would help..

One does not have a way of determining the current processes priority? Just need to do an alloc?

If we had the current processes "rank" then we could simply compare. If rank is okay give them the object. If not try to extend slab. If that succeeds clear the rank. If extending fails fail the alloc. There would be no need for a reserve slab.

What worries me about this whole thing is

1. It is designed to fail an allocation rather than guarantee that all succeed. Is it not possible to figure out which processes are not essential and simply put them to sleep until the situation clear up?

2. It seems to be based on global ordering of allocations which is not possible given large systems and the relativistic constraints of physics. Ordering of events get more expensive the bigger the system is.

How does this system work if you can just order events within a processor? Or within a node? Within a zone?

3. I do not see how this integrates with other allocation constraints:
DMA constraints, cpuset constraints, memory node constraints, GFP_THISNODE, MEMALLOC, GFP_HIGH.

Quoted 5 lines

2. It seems to be based on global ordering of allocations which is not possible given large systems and the relativistic constraints of physics. Ordering of events get more expensive the bigger the system is.

Quoted 2 lines

How does this system work if you can just order events within a processor? Or within a node? Within a zone?

/me fails again..

Its about ensuring ALLOC_NO_WATERMARKS memory only reaches PF_MEMALLOC processes, not joe random's pi calculator.

Quoted 8 lines · Peter Zijlstra

Quoted 7 lines

2. It seems to be based on global ordering of allocations which is not possible given large systems and the relativistic constraints of physics. Ordering of events get more expensive the bigger the system is.

How does this system work if you can just order events within a processor? Or within a node? Within a zone?

Quoted 4 lines

/me fails again..

Its about ensuring ALLOC_NO_WATERMARKS memory only reaches PF_MEMALLOC processes, not joe random's pi calculator.

Watermarks are per zone?

Quoted 4 lines

Quoted 2 lines

Its about ensuring ALLOC_NO_WATERMARKS memory only reaches PF_MEMALLOC processes, not joe random's pi calculator.

Watermarks are per zone?

Yes, but the page allocator might address multiple zones in order to obtain a page.

If I understand this correctly:

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool
do_io()

                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       ...
                                          eats it all

kmem_cache_alloc(...) -> ENOMEM
who ate my donuts?!

Quoted 13 lines · On Mon, 14 May 2007 11:12:24 -0500 Matt Mackall <[email protected]> wrote:

If I understand this correctly:

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool
do_io()

                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       ...
                                          eats it all

Quoted 2 lines

kmem_cache_alloc(...) -> ENOMEM
who ate my donuts?!

Yes, that's my understanding also.

I can see why it's a problem in theory, but I don't think Peter has yet revealed to us why it's a problem in practice. I got all excited when Christoph asked "I am not sure what the point of all of this is.", but Peter cunningly avoided answering that ;)

What observed problem is being fixed here?

Quoted 13 lines · On Mon, 14 May 2007 11:12:24 -0500 Matt Mackall <[email protected]> wrote:

If I understand this correctly:

privileged thread unprivileged greedy process
kmem_cache_alloc(...)
adds new slab page from lowmem pool
do_io()

                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       kmem_cache_alloc(...)
                                       ...
                                          eats it all

Quoted 2 lines

kmem_cache_alloc(...) -> ENOMEM
who ate my donuts?!

Yes, that's my understanding also.

I can see why it's a problem in theory, but I don't think Peter has yet revealed to us why it's a problem in practice. I got all excited when Christoph asked "I am not sure what the point of all of this is.", but Peter cunningly avoided answering that ;)

What observed problem is being fixed here?

In the interest of creating a reserve based allocator; we need to make the slab allocator (*sigh*, all three) fair with respect to GFP flags.

That is, we need to protect memory from being used by easier gfp flags than it was allocated with. If our reserve is placed below GFP_ATOMIC, we do not want a GFP_KERNEL allocation to walk away with it - a scenario that is perfectly possible with the current allocators.

Quoted 4 lines · Peter Zijlstra

In the interest of creating a reserve based allocator; we need to make the slab allocator (*sigh*, all three) fair with respect to GFP flags.

Quoted 4 lines

That is, we need to protect memory from being used by easier gfp flags than it was allocated with. If our reserve is placed below GFP_ATOMIC, we do not want a GFP_KERNEL allocation to walk away with it - a scenario that is perfectly possible with the current allocators.

And the solution is to fail the allocation of the process which tries to walk away with it. The failing allocation will lead to the killing of the process right?

Could you please modify the patchset to *avoid* failure conditions. This patchset here only manages failure conditions. The system should not get into the failure conditions in the first place! For that purpose you may want to put processes to sleep etc. But in order to do so you need to figure out which processes you need to make progress.

I'm really not seeing why you're making such a fuzz about it; normally when you push the system this hard we're failing allocations left right and center too. Its just that the block IO path has some mempools which allow it to write out some (swap) pages and slowly get back to sanity.

Quoted 4 lines · Peter Zijlstra

I'm really not seeing why you're making such a fuzz about it; normally when you push the system this hard we're failing allocations left right and center too. Its just that the block IO path has some mempools which allow it to write out some (swap) pages and slowly get back to sanity.

I am weirdly confused by these patches. Among other things you told me that the performance does not matter since its never (or rarely) being used (why do it then?).

Quoted 3 lines

I am weirdly confused by these patches. Among other things you told me that the performance does not matter since its never (or rarely) being used (why do it then?).

When we are very low on memory and do access the reserves by means of ALLOC_NO_WATERMARKS, we want to avoid processed that are not entitled to use such memory from running away with the little we have.

Quoted 4 lines · Peter Zijlstra

I'm really not seeing why you're making such a fuzz about it; normally when you push the system this hard we're failing allocations left right and center too. Its just that the block IO path has some mempools which allow it to write out some (swap) pages and slowly get back to sanity.

I am weirdly confused by these patches. Among other things you told me that the performance does not matter since its never (or rarely) being used (why do it then?).

Simply stated, the problem is sometimes it's impossible to free memory without allocating more memory. Thus we must keep enough protected reserve that we can guarantee progress. This is what mempools are for in the regular I/O stack. Unfortunately, mempools are a bad match for network I/O.

It's absolutely correct that performance doesn't matter in the case this patch is addressing. All that matters is digging ourselves out of OOM. The box either survives the crisis or it doesn't.

It's also correct that we should hardly ever get into a situation where we trigger this problem. But such cases are still fairly easy to trigger in some workloads. Swap over network is an excellent example, because we typically don't start swapping heavily until we're quite low on freeable memory.

Quoted 6 lines · Matt Mackall

Simply stated, the problem is sometimes it's impossible to free memory without allocating more memory. Thus we must keep enough protected reserve that we can guarantee progress. This is what mempools are for in the regular I/O stack. Unfortunately, mempools are a bad match for network I/O.

Quoted 3 lines

It's absolutely correct that performance doesn't matter in the case this patch is addressing. All that matters is digging ourselves out of OOM. The box either survives the crisis or it doesn't.

Well we fail allocations in order to do so and these allocations may be even nonatomic allocs. Pretty dangerous approach.

Quoted 5 lines

It's also correct that we should hardly ever get into a situation where we trigger this problem. But such cases are still fairly easy to trigger in some workloads. Swap over network is an excellent example, because we typically don't start swapping heavily until we're quite low on freeable memory.

Is it not possible to avoid failing allocs? Instead put processes to sleep? Run synchrononous reclaim?

The proposed patch doesn't change how the kernel functions at this point; it just enforces an existing rule better.

Quoted 2 lines · Peter Zijlstra

The proposed patch doesn't change how the kernel functions at this point; it just enforces an existing rule better.

Well I'd say it controls the allocation failures. And that only works if one can consider the system having a single zone.

Lets say the system has two cpusets A and B. A allocs from node 1 and B allocs from node 2. Two processes one in A and one in B run on the same processor.

Node 1 gets very low in memory so your patch kicks in and sets up the global memory emergency situation with the reserve slab.

Now the process in B will either fail although it has plenty of memory on node 2.

Or it may just clear the emergency slab and then the next critical alloc of the process in A that is low on memory will fail.

The way I read the cpuset page allocator, it will only respect the cpuset if there is memory aplenty. Otherwise it will grab whatever. So still, it will only ever use ALLOC_NO_WATERMARKS if the whole system is in distress.

The way I read the cpuset page allocator, it will only respect the cpuset if there is memory aplenty. Otherwise it will grab whatever. So still, it will only ever use ALLOC_NO_WATERMARKS if the whole system is in distress.

Quoted 4 lines · Peter Zijlstra

The way I read the cpuset page allocator, it will only respect the cpuset if there is memory aplenty. Otherwise it will grab whatever. So still, it will only ever use ALLOC_NO_WATERMARKS if the whole system is in distress.

Sorry no. The purpose of the cpuset is to limit memory for an application. If the boundaries would be fluid then we would not need cpusets.

But the same principles also apply for allocations to different zones in a SMP system. There are 4 zones DMA DMA32 NORMAL and HIGHMEM and we have general slabs for DMA and NORMAL. A slab that uses zone NORMAL falls back to DMA32 and DMA depending on the watermarks of the 3 zones. So a ZONE_NORMAL slab can exhaust memory available for ZONE_DMA.

Again the question is the watermarks of which zone? In case of the ZONE_NORMAL allocation you have 3 to pick from. Its the last one? Then its the same as ZONE_DMA, and you got a collision with the corresponding DMA slab. Depending the system deciding on a zone where we allocate the page from you may get a different watermark situation.

On x86_64 systems you have the additional complication that there are even multiple DMA32 or NORMAL zones per node. Some will have DMA32 and NORMAL, others DMA32 alone or NORMAL alone. Which watermarks are we talking about?

The use of ALLOC_NO_WATERMARKS depends on the contraints of the allocation in all cases. You can only compare the stresslevel (rank?) of allocations that have the same allocation constraints. The allocation constraints are a result of gfp flags,

cpuset configuration and memory policies in effect.

cpusets are ignored when in dire straights for an kernel alloc.

On Thu, 2007-05-17 at 15:27 -0700, Christoph Lameter wrote:

Isn't the zone mask the same for all allocations from a specific slab? If so, then the slab wide ->reserve_slab will still dtrt (barring cpusets).

Quoted 4 lines

On x86_64 systems you have the additional complication that there are even multiple DMA32 or NORMAL zones per node. Some will have DMA32 and NORMAL, others DMA32 alone or NORMAL alone. Which watermarks are we talking about?

Watermarks like used by the page allocator given the slabs zone mask. The page allocator will only fall back to ALLOC_NO_WATERMARKS when all target zones are exhausted.

Quoted 4 lines

The use of ALLOC_NO_WATERMARKS depends on the contraints of the allocation in all cases. You can only compare the stresslevel (rank?) of allocations that have the same allocation constraints. The allocation constraints are a result of gfp flags,

The gfp zone mask is constant per slab, no? It has to, because the zone mask is only used when the slab is extended, other allocations live off whatever was there before them.

Quoted 1 line

cpuset configuration and memory policies in effect.

Yes, I see now that these might become an issue, I will have to think on this.

I care about kernel allocations only. In particular about those that have PF_MEMALLOC semantics.

- set page->reserve nonzero for each page allocated with ALLOC_NO_WATERMARKS; which by the previous point implies that all available zones are below ALLOC_MIN|ALLOC_HIGH|ALLOC_HARDER

- when a page->reserve slab is allocated store it in s->reserve_slab and do not update the ->cpu_slab[] (this forces subsequent allocs to retry the allocation).

All ALLOC_NO_WATERMARKS enabled slab allocations are served from ->reserve_slab, up until the point where a !page->reserve slab alloc succeeds, at which point the ->reserve_slab is pushed into the partial lists and ->reserve_slab set to NULL.

Since only the allocation of a new slab uses the gfp zone flags, and other allocations placement hints they have to be uniform over all slab allocs for a given kmem_cache. Thus the s->reserve_slab/page->reserve status is kmem_cache wide.

Quoted 2 lines · Peter Zijlstra

I care about kernel allocations only. In particular about those that have PF_MEMALLOC semantics.

Hmmmm.. I wish I was more familiar with PF_MEMALLOC. ccing Nick.

Quoted 3 lines

- set page->reserve nonzero for each page allocated with ALLOC_NO_WATERMARKS; which by the previous point implies that all available zones are below ALLOC_MIN|ALLOC_HIGH|ALLOC_HARDER

Ok that adds a new field to the page struct. I suggested a page flag in slub before.

Quoted 3 lines

- when a page->reserve slab is allocated store it in s->reserve_slab and do not update the ->cpu_slab[] (this forces subsequent allocs to retry the allocation).

Right that should work.

Quoted 4 lines

All ALLOC_NO_WATERMARKS enabled slab allocations are served from ->reserve_slab, up until the point where a !page->reserve slab alloc succeeds, at which point the ->reserve_slab is pushed into the partial lists and ->reserve_slab set to NULL.

So the original issue is still not fixed. A slab alloc may succeed without watermarks if that particular allocation is restricted to a different set of nodes. Then the reserve slab is dropped despite the memory scarcity on another set of nodes?

Quoted 4 lines

Since only the allocation of a new slab uses the gfp zone flags, and other allocations placement hints they have to be uniform over all slab allocs for a given kmem_cache. Thus the s->reserve_slab/page->reserve status is kmem_cache wide.

No the gfp zone flags are not uniform and placement of page allocator allocs through SLUB do not always have the same allocation constraints.

SLUB will check the node of the page that was allocated when the page allocator returns and put the page into that nodes slab list. This varies depending on the allocation context.

Allocations can be particular to uses of a slab in particular situations. A kmalloc cache can be used to allocate from various sets of nodes in different circumstances. kmalloc will allow serving a limited number of objects from the wrong nodes for performance reasons but the next allocation from the page allocator (or from the partial lists) will occur using the current set of allowed nodes in order to ensure a rough obedience to the memory policies and cpusets. kmalloc_node behaves differently and will enforce using memory from a particular node.

From what I can see, it takes pretty much any page it can get once you

Quoted 4 lines

So the original issue is still not fixed. A slab alloc may succeed without watermarks if that particular allocation is restricted to a different set of nodes. Then the reserve slab is dropped despite the memory scarcity on another set of nodes?

I can't see how. This extra ALLOC_MIN|ALLOC_HIGH|ALLOC_HARDER alloc will first deplete all other zones. Once that starts failing no node should still have pages accessible by any allocation context other than PF_MEMALLOC.

Quoted 2 lines

No the gfp zone flags are not uniform and placement of page allocator allocs through SLUB do not always have the same allocation constraints.

It has to; since it can serve the allocation from a pre-existing slab allocation. Hence any page allocation must be valid for all other users.

Quoted 3 lines

SLUB will check the node of the page that was allocated when the page allocator returns and put the page into that nodes slab list. This varies depending on the allocation context.

Yes, it keeps slabs on per node lists. I'm just not seeing how this puts hard constraints on the allocations.

As far as I can see there cannot be a hard constraint here, because allocations form interrupt context are at best node local. And node affine zone lists still have all zones, just ordered on locality.

Quoted 8 lines

Allocations can be particular to uses of a slab in particular situations. A kmalloc cache can be used to allocate from various sets of nodes in different circumstances. kmalloc will allow serving a limited number of objects from the wrong nodes for performance reasons but the next allocation from the page allocator (or from the partial lists) will occur using the current set of allowed nodes in order to ensure a rough obedience to the memory policies and cpusets. kmalloc_node behaves differently and will enforce using memory from a particular node.

Quoted 1 line

From what I can see, it takes pretty much any page it can get once you

hit it with PF_MEMALLOC. If the page allocation doesn't use ALLOC_CPUSET the page can come from pretty much anywhere.

Quoted 5 lines · Peter Zijlstra

Quoted 4 lines

So the original issue is still not fixed. A slab alloc may succeed without watermarks if that particular allocation is restricted to a different set of nodes. Then the reserve slab is dropped despite the memory scarcity on another set of nodes?

Quoted 4 lines

I can't see how. This extra ALLOC_MIN|ALLOC_HIGH|ALLOC_HARDER alloc will first deplete all other zones. Once that starts failing no node should still have pages accessible by any allocation context other than PF_MEMALLOC.

This means we will disobey cpuset and memory policy constraints?

From what I can make of it, yes. Although I'm a bit hazy on the

Quoted 5 lines

Quoted 2 lines

No the gfp zone flags are not uniform and placement of page allocator allocs through SLUB do not always have the same allocation constraints.

It has to; since it can serve the allocation from a pre-existing slab allocation. Hence any page allocation must be valid for all other users.

Why does it have to? This is not true.

Quoted 6 lines

Quoted 3 lines

SLUB will check the node of the page that was allocated when the page allocator returns and put the page into that nodes slab list. This varies depending on the allocation context.

Yes, it keeps slabs on per node lists. I'm just not seeing how this puts hard constraints on the allocations.

The constraints come from the context of memory policies and cpusets. See get_any_partial().

Quoted 3 lines

As far as I can see there cannot be a hard constraint here, because allocations form interrupt context are at best node local. And node affine zone lists still have all zones, just ordered on locality.

Interrupt context is something different. If we do not have a process context then no cpuset and memory policy constraints can apply since we have no way of determining that. If you restrict your use of the reserve cpuset to only interrupt allocs then we may indeed be fine.

Quoted 9 lines

Quoted 8 lines

Allocations can be particular to uses of a slab in particular situations. A kmalloc cache can be used to allocate from various sets of nodes in different circumstances. kmalloc will allow serving a limited number of objects from the wrong nodes for performance reasons but the next allocation from the page allocator (or from the partial lists) will occur using the current set of allowed nodes in order to ensure a rough obedience to the memory policies and cpusets. kmalloc_node behaves differently and will enforce using memory from a particular node.

Quoted 3 lines

Quoted 1 line

From what I can see, it takes pretty much any page it can get once you

hit it with PF_MEMALLOC. If the page allocation doesn't use ALLOC_CPUSET the page can come from pretty much anywhere.

No it cannot. One the current cpuslab is exhaused (which can be anytime) it will enforce the contextual allocation constraints. See get_any_partial() in slub.c.

Quoted 3 lines

This means we will disobey cpuset and memory policy constraints?

From what I can make of it, yes. Although I'm a bit hazy on the

mempolicy code.

Quoted 6 lines

Quoted 5 lines

Quoted 2 lines

No the gfp zone flags are not uniform and placement of page allocator allocs through SLUB do not always have the same allocation constraints.

It has to; since it can serve the allocation from a pre-existing slab allocation. Hence any page allocation must be valid for all other users.

Quoted 1 line

Why does it have to? This is not true.

Say the slab gets allocated by an allocation from interrupt context; no cpuset, no policy. This same slab must be valid for whatever allocation comes next, right? Regardless of whatever policy or GFP_ flags are in effect for that allocation.

Quoted 2 lines

The constraints come from the context of memory policies and cpusets. See get_any_partial().

but get_partial() will only be called if the cpu_slab is full, up until that point you have to do with whatever is there.

Quoted 4 lines

Quoted 3 lines

As far as I can see there cannot be a hard constraint here, because allocations form interrupt context are at best node local. And node affine zone lists still have all zones, just ordered on locality.

Quoted 4 lines

Interrupt context is something different. If we do not have a process context then no cpuset and memory policy constraints can apply since we have no way of determining that. If you restrict your use of the reserve cpuset to only interrupt allocs then we may indeed be fine.

No, what I'm saying is that if the slab gets refilled from interrupt context the next process context alloc will have to work with whatever the interrupt left behind. Hence there is no hard constraint.

Quoted 7 lines

Quoted 3 lines

Quoted 1 line

From what I can see, it takes pretty much any page it can get once you

hit it with PF_MEMALLOC. If the page allocation doesn't use ALLOC_CPUSET the page can come from pretty much anywhere.

No it cannot. One the current cpuslab is exhaused (which can be anytime) it will enforce the contextual allocation constraints. See get_any_partial() in slub.c.

If it finds no partial slabs it goes back to the page allocator; and when you allocate a page under PF_MEMALLOC and the normal allocations are exhausted it takes a page from pretty much anywhere.

Quoted 4 lines · Peter Zijlstra

Quoted 3 lines

This means we will disobey cpuset and memory policy constraints?

From what I can make of it, yes. Although I'm a bit hazy on the

mempolicy code.

In an interrupt context we do not have a process context. But there is no exemption from memory policy constraints.

Quoted 8 lines

Quoted 7 lines

Quoted 5 lines

Quoted 2 lines

No the gfp zone flags are not uniform and placement of page allocator allocs through SLUB do not always have the same allocation constraints.

It has to; since it can serve the allocation from a pre-existing slab allocation. Hence any page allocation must be valid for all other users.

Why does it have to? This is not true.

Quoted 4 lines

Say the slab gets allocated by an allocation from interrupt context; no cpuset, no policy. This same slab must be valid for whatever allocation comes next, right? Regardless of whatever policy or GFP_ flags are in effect for that allocation.

Yes sure if we do not have a context then no restrictions originating there can be enforced. So you want to restrict the logic now to interrupt allocs? I.e. GFP_ATOMIC?

Quoted 5 lines

Quoted 2 lines

The constraints come from the context of memory policies and cpusets. See get_any_partial().

but get_partial() will only be called if the cpu_slab is full, up until that point you have to do with whatever is there.

Correct. That is an optimization but it may be called anytime from the perspective of an execution thread and that may cause problems with your approach.

Quoted 9 lines

Quoted 8 lines

Quoted 3 lines

As far as I can see there cannot be a hard constraint here, because allocations form interrupt context are at best node local. And node affine zone lists still have all zones, just ordered on locality.

Interrupt context is something different. If we do not have a process context then no cpuset and memory policy constraints can apply since we have no way of determining that. If you restrict your use of the reserve cpuset to only interrupt allocs then we may indeed be fine.

Quoted 3 lines

No, what I'm saying is that if the slab gets refilled from interrupt context the next process context alloc will have to work with whatever the interrupt left behind. Hence there is no hard constraint.

It will work with whatever was left behind in the case of SLUB and a kmalloc alloc (optimization there). It wont if its SLAB (which is stricter) or a kmalloc_node alloc. A kmalloc_node alloc will remove the current cpuslab if its not on the right now.

Quoted 11 lines

Quoted 7 lines

Quoted 3 lines

Quoted 1 line

From what I can see, it takes pretty much any page it can get once you

hit it with PF_MEMALLOC. If the page allocation doesn't use ALLOC_CPUSET the page can come from pretty much anywhere.

No it cannot. One the current cpuslab is exhaused (which can be anytime) it will enforce the contextual allocation constraints. See get_any_partial() in slub.c.

If it finds no partial slabs it goes back to the page allocator; and when you allocate a page under PF_MEMALLOC and the normal allocations are exhausted it takes a page from pretty much anywhere.

If it finds no partial slab then it will go to the page allocator which will allocate given the current contextual alloc constraints.

In the case
of a memory policy we may have limited the allocations to a single node where there is no escape (the zonelist does *not* contain zones of other nodes).

The only chance to bypass this is by only dealing with allocations during interrupt that have no allocation context.

Quoted 3 lines

Yes sure if we do not have a context then no restrictions originating there can be enforced. So you want to restrict the logic now to interrupt allocs? I.e. GFP_ATOMIC?

No, any kernel alloc.

Quoted 3 lines

Correct. That is an optimization but it may be called anytime from the perspective of an execution thread and that may cause problems with your approach.

I'm not seeing how this would interfere; if the alloc can be handled from a partial slab, that is fine.

Quoted 4 lines

In the case
of a memory policy we may have limited the allocations to a single node where there is no escape (the zonelist does *not* contain zones of other nodes).

Ah, this is the point I was missing; I assumed each zonelist would always include all zones, but would just continue/break the loop using things like cpuset_zone_allwed_*().

This might indeed foil the game.

I could 'fix' this by doing the PF_MEMALLOC allocation from the regular node zonelist instead of from the one handed down....

/me thinks out loud.. since direct reclaim runs in whatever process context was handed out we're stuck with whatever policy we started from; but since the allocations are kernel allocs - not userspace allocs, and we're in dire straights, it makes sense to violate the tasks restraints in order to keep the machine up.

memory policies are the only ones with 'short' zonelists, right? CPU sets are on top of whatever zonelist is handed out, and the normal zonelists include all nodes - ordered by distance

Quoted 2 lines

The only chance to bypass this is by only dealing with allocations during interrupt that have no allocation context.

But you just said that interrupts are not exempt from memory policies, and policies are the only ones that have 'short' zonelists. /me confused.