Skip to contentSkip to conversation

mm/page_alloc: unify __alloc_frozen_pages[_nolock]_noprof()

PATCH Superseded Brendan Jackman 6 participants 15 messages Jun 17 – Jun 22

Subsystems (7)

v1 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mm/page_alloc: unify __alloc_frozen_pages[_nolock]_noprof()

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

No functional change intended.

---
 mm/hugetlb.c    |   2 +-
 mm/internal.h   |   4 +-
 mm/mempolicy.c  |   8 +--
 mm/page_alloc.c | 175 +++++++++++++++++++++++++++++---------------------------
 mm/slub.c       |   4 +-
 5 files changed, 99 insertions(+), 94 deletions(-)
mm/hugetlb.c +1 -1
index 571212b80835e..619f6307dc98d 100644
@@ -1806,7 +1806,7 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
 	if (alloc_try_hard)
 		gfp_mask |= __GFP_RETRY_MAYFAIL;
 
-	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask);
+	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, 0);
 
 	/*
 	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
mm/internal.h +2 -2
index 181e79f1d6a20..1043eb833836c 100644
@@ -913,7 +913,7 @@ extern bool free_pages_prepare(struct page *page, unsigned int order);
 extern int user_min_free_kbytes;
 
 struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
-		nodemask_t *);
+		nodemask_t *, unsigned int alloc_flags);
 #define __alloc_frozen_pages(...) \
 	alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
 void free_frozen_pages(struct page *page, unsigned int order);
@@ -924,7 +924,7 @@ struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
 #else
 static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
 {
-	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
+	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, 0);
 }
 #endif
 
mm/mempolicy.c +4 -4
index 36699fabd3c22..dccff90682035 100644
@@ -2425,9 +2425,9 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
 	 */
 	preferred_gfp = gfp | __GFP_NOWARN;
 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
-	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, 0);
 	if (!page)
-		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
+		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, 0);
 
 	return page;
 }
@@ -2475,7 +2475,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 			 */
 			page = __alloc_frozen_pages_noprof(
 				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
-				nid, NULL);
+				nid, NULL, 0);
 			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
 				return page;
 			/*
@@ -2487,7 +2487,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 		}
 	}
 
-	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, 0);
 
 	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
 		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
mm/page_alloc.c +90 -85

217 lines not on this page · raw message

mm/slub.c +2 -2
index a2bf3756ca7d0..b9fb66071bd07 100644
@@ -3275,7 +3275,7 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
 	else if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages(flags, order);
 	else
-		page = __alloc_frozen_pages(flags, order, node, NULL);
+		page = __alloc_frozen_pages(flags, order, node, NULL, 0);
 
 	if (!page)
 		return NULL;
@@ -5236,7 +5236,7 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 	if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages_noprof(flags, order);
 	else
-		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
+		page = __alloc_frozen_pages_noprof(flags, order, node, NULL, 0);
 
 	if (page) {
 		ptr = page_address(page);

---
base-commit: 1111012ec6508a38a39f8d20c213c8c9cf3c96c0
change-id: 20260617-alloc-trylock-14ad37dab337

Best regards,

Signature · 1 line
Brendan Jackman <[email protected]>
#1 lore raw

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

Quoted 3 lines

No functional change intended.

Besides the need to ponder unintended effects, mostly LGTM. Just not a fan of the hardcoded '0' passed at various places. In the slab variant of this (the thread I've linked above) I went with SLAB_ALLOC_DEFAULT, so you can do e.g. ALLOC_DEFAULT here?

Quoted 334 lines
---
 mm/hugetlb.c    |   2 +-
 mm/internal.h   |   4 +-
 mm/mempolicy.c  |   8 +--
 mm/page_alloc.c | 175 +++++++++++++++++++++++++++++---------------------------
 mm/slub.c       |   4 +-
 5 files changed, 99 insertions(+), 94 deletions(-)
mm/hugetlb.c +1 -1
index 571212b80835e..619f6307dc98d 100644
@@ -1806,7 +1806,7 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
 	if (alloc_try_hard)
 		gfp_mask |= __GFP_RETRY_MAYFAIL;
 
-	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask);
+	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, 0);
 
 	/*
 	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
mm/internal.h +2 -2
index 181e79f1d6a20..1043eb833836c 100644
@@ -913,7 +913,7 @@ extern bool free_pages_prepare(struct page *page, unsigned int order);
 extern int user_min_free_kbytes;
 
 struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
-		nodemask_t *);
+		nodemask_t *, unsigned int alloc_flags);
 #define __alloc_frozen_pages(...) \
 	alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
 void free_frozen_pages(struct page *page, unsigned int order);
@@ -924,7 +924,7 @@ struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
 #else
 static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
 {
-	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
+	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, 0);
 }
 #endif
 
mm/mempolicy.c +4 -4
index 36699fabd3c22..dccff90682035 100644
@@ -2425,9 +2425,9 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
 	 */
 	preferred_gfp = gfp | __GFP_NOWARN;
 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
-	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, 0);
 	if (!page)
-		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
+		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, 0);
 
 	return page;
 }
@@ -2475,7 +2475,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 			 */
 			page = __alloc_frozen_pages_noprof(
 				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
-				nid, NULL);
+				nid, NULL, 0);
 			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
 				return page;
 			/*
@@ -2487,7 +2487,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 		}
 	}
 
-	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, 0);
 
 	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
 		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
mm/page_alloc.c +90 -85
index 0111cdbdb5321..fc4d07bbf44b5 100644
@@ -5253,24 +5253,98 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
 	}
 }
 
-/*
- * This is the 'heart' of the zoned buddy allocator.
- */
-struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+				       unsigned int alloc_flags)
 {
-	struct page *page;
-	unsigned int alloc_flags = ALLOC_WMARK_LOW;
-	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-	struct alloc_context ac = { };
+
+	if (alloc_flags & ALLOC_TRYLOCK)
+		return pcp_allowed_order(order);
 
 	/*
 	 * There are several places where we assume that the order value is sane
 	 * so bail out early if the request is out of bound.
 	 */
-	if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp))
+	return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp));
+}
+
+static inline bool alloc_trylock_allowed(void)
+{
+	/*
+	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
+	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
+	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
+	 * mark the task as the owner of another rt_spin_lock which will
+	 * confuse PI logic, so return immediately if called from hard IRQ or
+	 * NMI.
+	 *
+	 * Note, irqs_disabled() case is ok. This function can be called
+	 * from raw_spin_lock_irqsave region.
+	 */
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
+		return false;
+
+	/* On UP, spin_trylock() always succeeds even when it is locked */
+	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+		return false;
+
+	/* Bailout, since _deferred_grow_zone() needs to take a lock */
+	if (deferred_pages_enabled())
+		return false;
+
+	return true;
+}
+
+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *
+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.
+ *
+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */
+static const gfp_t gfp_trylock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+				__GFP_COMP;
+
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+	struct page *page;
+	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+	struct alloc_context ac = { };
+
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
 		return NULL;
 
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
+		return NULL;
+
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);
+		if (!alloc_trylock_allowed())
+			return NULL;
+		gfp |= gfp_trylock;
+	} else {
+		alloc_flags |= ALLOC_WMARK_LOW;
+	}
+
 	gfp &= gfp_allowed_mask;
 	/*
 	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
@@ -5291,9 +5365,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 	 */
 	alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
 
-	/* First allocation attempt */
+	/* First allocation attempt (or, for trylock, only attempt) */
 	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-	if (likely(page))
+	if (likely(page) || (alloc_flags & ALLOC_TRYLOCK))
 		goto out;
 
 	alloc_gfp = gfp;
@@ -5310,7 +5384,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 out:
 	if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
 	    unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
-		free_frozen_pages(page, order);
+		__free_frozen_pages(page, order,
+				    alloc_flags & ALLOC_TRYLOCK ? FPI_TRYLOCK : 0);
 		page = NULL;
 	}
 
@@ -5326,7 +5401,7 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order,
 {
 	struct page *page;
 
-	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, 0);
 	if (page)
 		set_page_refcounted(page);
 	return page;
@@ -7856,80 +7931,10 @@ static bool __free_unaccepted(struct page *page)
 
 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
 {
-	/*
-	 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
-	 * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
-	 * is not safe in arbitrary context.
-	 *
-	 * These two are the conditions for gfpflags_allow_spinning() being true.
-	 *
-	 * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
-	 * to warn. Also warn would trigger printk() which is unsafe from
-	 * various contexts. We cannot use printk_deferred_enter() to mitigate,
-	 * since the running context is unknown.
-	 *
-	 * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
-	 * is safe in any context. Also zeroing the page is mandatory for
-	 * BPF use cases.
-	 *
-	 * Though __GFP_NOMEMALLOC is not checked in the code path below,
-	 * specify it here to highlight that alloc_pages_nolock()
-	 * doesn't want to deplete reserves.
-	 */
-	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
-			| gfp_flags;
-	unsigned int alloc_flags = ALLOC_TRYLOCK;
-	struct alloc_context ac = { };
-	struct page *page;
-
-	VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
-	/*
-	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
-	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
-	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
-	 * mark the task as the owner of another rt_spin_lock which will
-	 * confuse PI logic, so return immediately if called from hard IRQ or
-	 * NMI.
-	 *
-	 * Note, irqs_disabled() case is ok. This function can be called
-	 * from raw_spin_lock_irqsave region.
-	 */
-	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
-		return NULL;
-
-	/* On UP, spin_trylock() always succeeds even when it is locked */
-	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
-		return NULL;
-
-	if (!pcp_allowed_order(order))
-		return NULL;
-
-	/* Bailout, since _deferred_grow_zone() needs to take a lock */
-	if (deferred_pages_enabled())
-		return NULL;
-
 	if (nid == NUMA_NO_NODE)
 		nid = numa_node_id();
 
-	prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac,
-			    &alloc_gfp, &alloc_flags);
-
-	/*
-	 * Best effort allocation from percpu free list.
-	 * If it's empty attempt to spin_trylock zone->lock.
-	 */
-	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-
-	/* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */
-
-	if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
-	    unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
-		__free_frozen_pages(page, order, FPI_TRYLOCK);
-		page = NULL;
-	}
-	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
-	kmsan_alloc_page(page, order, alloc_gfp);
-	return page;
+	return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_TRYLOCK);
 }
 /**
  * alloc_pages_nolock - opportunistic reentrant allocation from any context
mm/slub.c +2 -3
index a2bf3756ca7d0..b9fb66071bd07 100644
@@ -3275,7 +3275,7 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
 	else if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages(flags, order);
 	else
-		page = __alloc_frozen_pages(flags, order, node, NULL);
+		page = __alloc_frozen_pages(flags, order, node, NULL, 0);
 
 	if (!page)
 		return NULL;
@@ -5236,7 +5236,7 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 	if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages_noprof(flags, order);
 	else
-		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
+		page = __alloc_frozen_pages_noprof(flags, order, node, NULL, 0);
 
 	if (page) {
 		ptr = page_address(page);
 
---

base-commit: 1111012ec6508a38a39f8d20c213c8c9cf3c96c0
change-id: 20260617-alloc-trylock-14ad37dab337

Best regards,

Signature · 1 line
Brendan Jackman <[email protected]>
#2 lore raw
Quoted 42 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

Quoted 7 lines

I'll have to ponder it more closely.

Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

[1] https://lore.kernel.org/all/[email protected]/

Quoted 345 lines
Quoted 4 lines

No functional change intended.

Besides the need to ponder unintended effects, mostly LGTM. Just not a fan of the hardcoded '0' passed at various places. In the slab variant of this (the thread I've linked above) I went with SLAB_ALLOC_DEFAULT, so you can do e.g. ALLOC_DEFAULT here?

Quoted 334 lines
---
 mm/hugetlb.c    |   2 +-
 mm/internal.h   |   4 +-
 mm/mempolicy.c  |   8 +--
 mm/page_alloc.c | 175 +++++++++++++++++++++++++++++---------------------------
 mm/slub.c       |   4 +-
 5 files changed, 99 insertions(+), 94 deletions(-)
mm/hugetlb.c +1 -1
index 571212b80835e..619f6307dc98d 100644
@@ -1806,7 +1806,7 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
      if (alloc_try_hard)
              gfp_mask |= __GFP_RETRY_MAYFAIL;
 
-     folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask);
+     folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, 0);
 
      /*
       * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
mm/internal.h +2 -2
index 181e79f1d6a20..1043eb833836c 100644
@@ -913,7 +913,7 @@ extern bool free_pages_prepare(struct page *page, unsigned int order);
 extern int user_min_free_kbytes;
 
 struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
-             nodemask_t *);
+             nodemask_t *, unsigned int alloc_flags);
 #define __alloc_frozen_pages(...) \
      alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
 void free_frozen_pages(struct page *page, unsigned int order);
@@ -924,7 +924,7 @@ struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
 #else
 static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
 {
-     return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
+     return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, 0);
 }
 #endif
 
mm/mempolicy.c +4 -4
index 36699fabd3c22..dccff90682035 100644
@@ -2425,9 +2425,9 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
       */
      preferred_gfp = gfp | __GFP_NOWARN;
      preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
-     page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
+     page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, 0);
      if (!page)
-             page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
+             page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, 0);
 
      return page;
 }
@@ -2475,7 +2475,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
                       */
                      page = __alloc_frozen_pages_noprof(
                              gfp | __GFP_THISNODE | __GFP_NORETRY, order,
-                             nid, NULL);
+                             nid, NULL, 0);
                      if (page || !(gfp & __GFP_DIRECT_RECLAIM))
                              return page;
                      /*
@@ -2487,7 +2487,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
              }
      }
 
-     page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
+     page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, 0);
 
      if (unlikely(pol->mode == MPOL_INTERLEAVE ||
                   pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
mm/page_alloc.c +90 -85
index 0111cdbdb5321..fc4d07bbf44b5 100644
@@ -5253,24 +5253,98 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
      }
 }
 
-/*
- * This is the 'heart' of the zoned buddy allocator.
- */
-struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-             int preferred_nid, nodemask_t *nodemask)
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+                                    unsigned int alloc_flags)
 {
-     struct page *page;
-     unsigned int alloc_flags = ALLOC_WMARK_LOW;
-     gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-     struct alloc_context ac = { };
+
+     if (alloc_flags & ALLOC_TRYLOCK)
+             return pcp_allowed_order(order);
 
      /*
       * There are several places where we assume that the order value is sane
       * so bail out early if the request is out of bound.
       */
-     if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp))
+     return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp));
+}
+
+static inline bool alloc_trylock_allowed(void)
+{
+     /*
+      * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
+      * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
+      * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
+      * mark the task as the owner of another rt_spin_lock which will
+      * confuse PI logic, so return immediately if called from hard IRQ or
+      * NMI.
+      *
+      * Note, irqs_disabled() case is ok. This function can be called
+      * from raw_spin_lock_irqsave region.
+      */
+     if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
+             return false;
+
+     /* On UP, spin_trylock() always succeeds even when it is locked */
+     if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+             return false;
+
+     /* Bailout, since _deferred_grow_zone() needs to take a lock */
+     if (deferred_pages_enabled())
+             return false;
+
+     return true;
+}
+
+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *
+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.
+ *
+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */
+static const gfp_t gfp_trylock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+                             __GFP_COMP;
+
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+             int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+     struct page *page;
+     gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+     struct alloc_context ac = { };
+
+     /* Other flags could be supported later if needed. */
+     if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
              return NULL;
 
+     if (!alloc_order_allowed(gfp, order, alloc_flags))
+             return NULL;
+
+     if (alloc_flags & ALLOC_TRYLOCK) {
+             VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);
+             if (!alloc_trylock_allowed())
+                     return NULL;
+             gfp |= gfp_trylock;
+     } else {
+             alloc_flags |= ALLOC_WMARK_LOW;
+     }
+
      gfp &= gfp_allowed_mask;
      /*
       * Apply scoped allocation constraints. This is mainly about GFP_NOFS
@@ -5291,9 +5365,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
       */
      alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
 
-     /* First allocation attempt */
+     /* First allocation attempt (or, for trylock, only attempt) */
      page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-     if (likely(page))
+     if (likely(page) || (alloc_flags & ALLOC_TRYLOCK))
              goto out;
 
      alloc_gfp = gfp;
@@ -5310,7 +5384,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 out:
      if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
          unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
-             free_frozen_pages(page, order);
+             __free_frozen_pages(page, order,
+                                 alloc_flags & ALLOC_TRYLOCK ? FPI_TRYLOCK : 0);
              page = NULL;
      }
 
@@ -5326,7 +5401,7 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order,
 {
      struct page *page;
 
-     page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask);
+     page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, 0);
      if (page)
              set_page_refcounted(page);
      return page;
@@ -7856,80 +7931,10 @@ static bool __free_unaccepted(struct page *page)
 
 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
 {
-     /*
-      * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
-      * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
-      * is not safe in arbitrary context.
-      *
-      * These two are the conditions for gfpflags_allow_spinning() being true.
-      *
-      * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
-      * to warn. Also warn would trigger printk() which is unsafe from
-      * various contexts. We cannot use printk_deferred_enter() to mitigate,
-      * since the running context is unknown.
-      *
-      * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
-      * is safe in any context. Also zeroing the page is mandatory for
-      * BPF use cases.
-      *
-      * Though __GFP_NOMEMALLOC is not checked in the code path below,
-      * specify it here to highlight that alloc_pages_nolock()
-      * doesn't want to deplete reserves.
-      */
-     gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
-                     | gfp_flags;
-     unsigned int alloc_flags = ALLOC_TRYLOCK;
-     struct alloc_context ac = { };
-     struct page *page;
-
-     VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
-     /*
-      * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
-      * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
-      * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
-      * mark the task as the owner of another rt_spin_lock which will
-      * confuse PI logic, so return immediately if called from hard IRQ or
-      * NMI.
-      *
-      * Note, irqs_disabled() case is ok. This function can be called
-      * from raw_spin_lock_irqsave region.
-      */
-     if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
-             return NULL;
-
-     /* On UP, spin_trylock() always succeeds even when it is locked */
-     if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
-             return NULL;
-
-     if (!pcp_allowed_order(order))
-             return NULL;
-
-     /* Bailout, since _deferred_grow_zone() needs to take a lock */
-     if (deferred_pages_enabled())
-             return NULL;
-
      if (nid == NUMA_NO_NODE)
              nid = numa_node_id();
 
-     prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac,
-                         &alloc_gfp, &alloc_flags);
-
-     /*
-      * Best effort allocation from percpu free list.
-      * If it's empty attempt to spin_trylock zone->lock.
-      */
-     page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-
-     /* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */
-
-     if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
-         unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
-             __free_frozen_pages(page, order, FPI_TRYLOCK);
-             page = NULL;
-     }
-     trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
-     kmsan_alloc_page(page, order, alloc_gfp);
-     return page;
+     return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_TRYLOCK);
 }
 /**
  * alloc_pages_nolock - opportunistic reentrant allocation from any context
mm/slub.c +2 -3
index a2bf3756ca7d0..b9fb66071bd07 100644
@@ -3275,7 +3275,7 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
      else if (node == NUMA_NO_NODE)
              page = alloc_frozen_pages(flags, order);
      else
-             page = __alloc_frozen_pages(flags, order, node, NULL);
+             page = __alloc_frozen_pages(flags, order, node, NULL, 0);
 
      if (!page)
              return NULL;
@@ -5236,7 +5236,7 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
      if (node == NUMA_NO_NODE)
              page = alloc_frozen_pages_noprof(flags, order);
      else
-             page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
+             page = __alloc_frozen_pages_noprof(flags, order, node, NULL, 0);
 
      if (page) {
              ptr = page_address(page);
 
---

base-commit: 1111012ec6508a38a39f8d20c213c8c9cf3c96c0
change-id: 20260617-alloc-trylock-14ad37dab337

Best regards,
--
Brendan Jackman <[email protected]>

#3 lore raw
Quoted 8 lines · Suren Baghdasaryan
Quoted 6 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 29 lines
Quoted 29 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 9 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Quoted 5 lines

Besides the need to ponder unintended effects, mostly LGTM. Just not a fan of the hardcoded '0' passed at various places. In the slab variant of this (the thread I've linked above) I went with SLAB_ALLOC_DEFAULT, so you can do e.g. ALLOC_DEFAULT here?

Yup ALLOC_DEFAULT sounds fine to me.

Thanks for the reviews as always.

#4 lore raw
Quoted 45 lines · Brendan Jackman
Quoted 6 lines · Suren Baghdasaryan
Quoted 4 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 28 lines
Quoted 28 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 4 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Hi Brendan, Suren,

Thanks for CC'ing me, Suren. This is indeed a viable approach

and I believe it brings us one step closer to removing

__GFP_NO_CODETAG entirely.

Brendan, I'd actually put together a rough local implementation

earlier with mostly the same core idea as yours, and this change

would indeed be minimal based on your patch.

Thanks a lot for being interested in tacking this into your v2 patch series.

Thanks

Best Regards

Hao

Quoted 7 lines
Quoted 4 lines
Quoted 4 lines

Besides the need to ponder unintended effects, mostly LGTM. Just not a fan of the hardcoded '0' passed at various places. In the slab variant of this (the thread I've linked above) I went with SLAB_ALLOC_DEFAULT, so you can do e.g. ALLOC_DEFAULT here?

Yup ALLOC_DEFAULT sounds fine to me.

Thanks for the reviews as always.

#5 lore raw
Quoted 70 lines · Hao Ge
Quoted 45 lines · Brendan Jackman
Quoted 6 lines · Suren Baghdasaryan
Quoted 4 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 28 lines
Quoted 28 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 4 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Hi Brendan, Suren,

Thanks for CC'ing me, Suren. This is indeed a viable approach

and I believe it brings us one step closer to removing

__GFP_NO_CODETAG entirely.

Quoted 7 lines

Brendan, I'd actually put together a rough local implementation

earlier with mostly the same core idea as yours, and this change

would indeed be minimal based on your patch.

Thanks a lot for being interested in tacking this into your v2 patch series.

Oh, I just took a look and it's a bit more fiddly than I thought because alloc_tag.c is actually in lib/ not mm/.

How did you tackle that, can you share your implementation? It would be nice if we can avoid exposing alloc_flags in gfp.h.

#6 lore raw
Quoted 74 lines · Brendan Jackman
Quoted 72 lines · Hao Ge
Quoted 45 lines · Brendan Jackman
Quoted 6 lines · Suren Baghdasaryan
Quoted 4 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 28 lines
Quoted 28 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 4 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Hi Brendan, Suren,

Thanks for CC'ing me, Suren. This is indeed a viable approach

and I believe it brings us one step closer to removing

__GFP_NO_CODETAG entirely.

Brendan, I'd actually put together a rough local implementation

Quoted 8 lines
Quoted 5 lines

earlier with mostly the same core idea as yours, and this change

would indeed be minimal based on your patch.

Thanks a lot for being interested in tacking this into your v2 patch series.

Oh, I just took a look and it's a bit more fiddly than I thought because alloc_tag.c is actually in lib/ not mm/.

One option is to move alloc_tag.c into mm/ (while keeping more generic codetag.c in lib/). From a quick look, that seems doable and probably the easiest approach.

Quoted 3 lines

How did you tackle that, can you share your implementation? It would be nice if we can avoid exposing alloc_flags in gfp.h.

#7 lore raw
Quoted 73 lines · Suren Baghdasaryan
Quoted 71 lines · Brendan Jackman
Quoted 70 lines · Hao Ge
Quoted 45 lines · Brendan Jackman
Quoted 6 lines · Suren Baghdasaryan
Quoted 4 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 28 lines
Quoted 28 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 4 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Hi Brendan, Suren,

Thanks for CC'ing me, Suren. This is indeed a viable approach

and I believe it brings us one step closer to removing

__GFP_NO_CODETAG entirely.

Brendan, I'd actually put together a rough local implementation

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

earlier with mostly the same core idea as yours, and this change

would indeed be minimal based on your patch.

Thanks a lot for being interested in tacking this into your v2 patch series.

Oh, I just took a look and it's a bit more fiddly than I thought because alloc_tag.c is actually in lib/ not mm/.

Hi Suren and Bredan

Quoted 6 lines

One option is to move alloc_tag.c into mm/ (while keeping more generic codetag.c in lib/). From a quick look, that seems doable and probably the easiest approach.

Quoted 2 lines

How did you tackle that, can you share your implementation? It would be nice if we can avoid exposing alloc_flags in gfp.h.

First, I introduced the ALLOC_NO_CODETAG flag as shown below:

@@ -1478,6 +1480,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, #define ALLOC_HIGHATOMIC       0x200 /* Allows access to MIGRATE_HIGHATOMIC */ #define ALLOC_TRYLOCK          0x400 /* Only use spin_trylock in allocation path */ #define ALLOC_KSWAPD           0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ +#define ALLOC_NO_CODETAG       0x1000 /* skip codetag tracking for this allocation */

Then, mirroring __alloc_pages_noprof, we wrapped a helper function named alloc_pages_noprof_notag.

@@ -5252,13 +5335,25 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, {
        struct page *page;

-       page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); +       page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, 0); if (page)
                set_page_refcounted(page);
        return page;
 }
 EXPORT_SYMBOL(__alloc_pages_noprof);

+struct page *alloc_pages_noprof_notag(gfp_t gfp, unsigned int order) +{
+       struct page *page;
+
+       page = __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, +                                          ALLOC_NO_CODETAG);
+       if (page)
+               set_page_refcounted(page);
+       return page;
+}
+EXPORT_SYMBOL_GPL(alloc_pages_noprof_notag);

Lastly, we exported this function in gfp.h as shown below:

include/linux/gfp.h +0 -0
index 51ef13ed756e..ac6e837ac8c0 100644
@@ -234,6 +234,9 @@ struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_

                nodemask_t *nodemask);
 #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__))

+struct page *alloc_pages_noprof_notag(gfp_t gfp, unsigned int order); +#define alloc_pages_notag(...) alloc_hooks(alloc_pages_noprof_notag(__VA_ARGS__))

Hope this information helps you.

Thanks

Best Regards

Hao

#8 lore raw
Quoted 156 lines · Hao Ge
Quoted 73 lines · Suren Baghdasaryan
Quoted 71 lines · Brendan Jackman
Quoted 70 lines · Hao Ge
Quoted 45 lines · Brendan Jackman
Quoted 6 lines · Suren Baghdasaryan
Quoted 4 lines · Vlastimil Babka (SUSE)

+Cc Alexei

Quoted 1 line · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the

Quoted 8 lines
Quoted 8 lines

It's not, it's ALLOC_TRYLOCK! Thanks for proving that we need to rename it to ALLOC_NOLOCK:

https://lore.kernel.org/all/[email protected]/

So you just won the job to do the rename :) I think it should be done before this patch, so that the new usages and other _trylock names introduced here can be done as _nolock outright.

Ack. I'll aim to send that tomorrow once Sashiko has caught up.

Quoted 28 lines
Quoted 28 lines
Quoted 18 lines

main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

Quoted 8 lines

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

I'll have to ponder it more closely.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

Ack.

I think this change might also help us in removing __GFP_NO_CODETAG

Nice, this actually looks trivial? I can probably just tack it onto the v2 for this patch/series.

Quoted 4 lines

introduced in [1] and being the only user of __GFP_NO_OBJ_EXT once Vlastimil's patchset removing other __GFP_NO_OBJ_EXT users lands. CC'ing Hao as he is brainstorming ways to remove __GFP_NO_CODETAG, and this might be the answer.

Hi Brendan, Suren,

Thanks for CC'ing me, Suren. This is indeed a viable approach

and I believe it brings us one step closer to removing

__GFP_NO_CODETAG entirely.

Brendan, I'd actually put together a rough local implementation

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

earlier with mostly the same core idea as yours, and this change

would indeed be minimal based on your patch.

Thanks a lot for being interested in tacking this into your v2 patch series.

Oh, I just took a look and it's a bit more fiddly than I thought because alloc_tag.c is actually in lib/ not mm/.

Hi Suren and Bredan

Quoted 6 lines

One option is to move alloc_tag.c into mm/ (while keeping more generic codetag.c in lib/). From a quick look, that seems doable and probably the easiest approach.

Quoted 2 lines

How did you tackle that, can you share your implementation? It would be nice if we can avoid exposing alloc_flags in gfp.h.

First, I introduced the ALLOC_NO_CODETAG flag as shown below:

(quoted diff) +0 -0
@@ -1478,6 +1480,7 @@ unsigned int reclaim_clean_pages_from_list(struct 
 zone *zone,
  #define ALLOC_HIGHATOMIC       0x200 /* Allows access to 

MIGRATE_HIGHATOMIC */
 #define ALLOC_TRYLOCK          0x400 /* Only use spin_trylock in
allocation path */
 #define ALLOC_KSWAPD           0x800 /* allow waking of kswapd,
__GFP_KSWAPD_RECLAIM set */
+#define ALLOC_NO_CODETAG       0x1000 /* skip codetag tracking for this allocation */

Then, mirroring __alloc_pages_noprof, we wrapped a helper function named alloc_pages_noprof_notag.

(quoted diff) +12 -1
@@ -5252,13 +5335,25 @@ struct page *__alloc_pages_noprof(gfp_t gfp, 
 unsigned int order,
  {
         struct page *page;
 
-       page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, 
 nodemask);
+       page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, 
 nodemask, 0);
         if (page)
                 set_page_refcounted(page);
         return page;
  }
  EXPORT_SYMBOL(__alloc_pages_noprof);
 
+struct page *alloc_pages_noprof_notag(gfp_t gfp, unsigned int order)
+{
+       struct page *page;
+
+       page = __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL,
+                                          ALLOC_NO_CODETAG);
+       if (page)
+               set_page_refcounted(page);
+       return page;
+}
+EXPORT_SYMBOL_GPL(alloc_pages_noprof_notag);

Lastly, we exported this function in gfp.h as shown below:

include/linux/gfp.h +0 -0
index 51ef13ed756e..ac6e837ac8c0 100644
@@ -234,6 +234,9 @@ struct folio *__folio_alloc_noprof(gfp_t gfp, 

unsigned int order, int preferred_
                nodemask_t *nodemask);
 #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__))

include/linux/gfp.h +2 -0
+struct page *alloc_pages_noprof_notag(gfp_t gfp, unsigned int order);
+#define alloc_pages_notag(...) 

alloc_hooks(alloc_pages_noprof_notag(__VA_ARGS__))

Quoted 1 line

Hope this information helps you.

Cool, thanks! On Friday I also tried doing it by just moving the .c file and that also seems to be pretty practical.

#9 lore raw
In reply to Brendan Jackman

Hi Brendan

Quoted 25 lines · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

Quoted 6 lines

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

If so, I believe we could generalize this further.

Under the current logic, |__alloc_pages_slowpath| cannot access the alloc_flags passed down from upper-level callers.
As I discussed in another thread, we can introduce a new alloc_flags to replace the |__GFP_NO_CODETAG| (|__GFP_NO_OBJ_EXT|) GFP flag.
This newly added flag needs to be propagated along the entire call chain down to |prep_new_page|, which means |__alloc_pages_slowpath| also has to handle this flag accordingly.

I'm wondering if we could introduce a caller_alloc_flags field within struct alloc_context to handle alloc_flags that need to persist throughout the entire page allocation cycle, when such flags exist. I'm sure others will have more appropriate solutions.

Thanks
Best Regards
Hao

Quoted 339 lines

No functional change intended.

---
  mm/hugetlb.c    |   2 +-
  mm/internal.h   |   4 +-
  mm/mempolicy.c  |   8 +--
  mm/page_alloc.c | 175 +++++++++++++++++++++++++++++---------------------------
  mm/slub.c       |   4 +-
  5 files changed, 99 insertions(+), 94 deletions(-)
mm/hugetlb.c +1 -1
index 571212b80835e..619f6307dc98d 100644
@@ -1806,7 +1806,7 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
  	if (alloc_try_hard)
  		gfp_mask |= __GFP_RETRY_MAYFAIL;
 
-	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask);
+	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask, 0);
 
  	/*
  	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
mm/internal.h +2 -2
index 181e79f1d6a20..1043eb833836c 100644
@@ -913,7 +913,7 @@ extern bool free_pages_prepare(struct page *page, unsigned int order);
  extern int user_min_free_kbytes;
 
  struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
-		nodemask_t *);
+		nodemask_t *, unsigned int alloc_flags);
  #define __alloc_frozen_pages(...) \
  	alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
  void free_frozen_pages(struct page *page, unsigned int order);
@@ -924,7 +924,7 @@ struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
  #else
  static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
  {
-	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
+	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL, 0);
  }
  #endif
 
mm/mempolicy.c +4 -4
index 36699fabd3c22..dccff90682035 100644
@@ -2425,9 +2425,9 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
  	 */
  	preferred_gfp = gfp | __GFP_NOWARN;
  	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
-	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, 0);
  	if (!page)
-		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
+		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, 0);
 
  	return page;
  }
@@ -2475,7 +2475,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
  			 */
  			page = __alloc_frozen_pages_noprof(
  				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
-				nid, NULL);
+				nid, NULL, 0);
  			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
  				return page;
  			/*
@@ -2487,7 +2487,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
  		}
  	}
 
-	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, 0);
 
  	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
  		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
mm/page_alloc.c +90 -85
index 0111cdbdb5321..fc4d07bbf44b5 100644
@@ -5253,24 +5253,98 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
  	}
  }
 
-/*
- * This is the 'heart' of the zoned buddy allocator.
- */
-struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+				       unsigned int alloc_flags)
  {
-	struct page *page;
-	unsigned int alloc_flags = ALLOC_WMARK_LOW;
-	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-	struct alloc_context ac = { };
+
+	if (alloc_flags & ALLOC_TRYLOCK)
+		return pcp_allowed_order(order);
 
  	/*
  	 * There are several places where we assume that the order value is sane
  	 * so bail out early if the request is out of bound.
  	 */
-	if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp))
+	return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp));
+}
+
+static inline bool alloc_trylock_allowed(void)
+{
+	/*
+	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
+	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
+	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
+	 * mark the task as the owner of another rt_spin_lock which will
+	 * confuse PI logic, so return immediately if called from hard IRQ or
+	 * NMI.
+	 *
+	 * Note, irqs_disabled() case is ok. This function can be called
+	 * from raw_spin_lock_irqsave region.
+	 */
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
+		return false;
+
+	/* On UP, spin_trylock() always succeeds even when it is locked */
+	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+		return false;
+
+	/* Bailout, since _deferred_grow_zone() needs to take a lock */
+	if (deferred_pages_enabled())
+		return false;
+
+	return true;
+}
+
+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *
+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.
+ *
+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */
+static const gfp_t gfp_trylock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+				__GFP_COMP;
+
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+	struct page *page;
+	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+	struct alloc_context ac = { };
+
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
  		return NULL;
 
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
+		return NULL;
+
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);
+		if (!alloc_trylock_allowed())
+			return NULL;
+		gfp |= gfp_trylock;
+	} else {
+		alloc_flags |= ALLOC_WMARK_LOW;
+	}
+
  	gfp &= gfp_allowed_mask;
  	/*
  	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
@@ -5291,9 +5365,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
  	 */
  	alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
 
-	/* First allocation attempt */
+	/* First allocation attempt (or, for trylock, only attempt) */
  	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-	if (likely(page))
+	if (likely(page) || (alloc_flags & ALLOC_TRYLOCK))
  		goto out;
 
  	alloc_gfp = gfp;
@@ -5310,7 +5384,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
  out:
  	if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
  	    unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
-		free_frozen_pages(page, order);
+		__free_frozen_pages(page, order,
+				    alloc_flags & ALLOC_TRYLOCK ? FPI_TRYLOCK : 0);
  		page = NULL;
  	}
 
@@ -5326,7 +5401,7 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order,
  {
  	struct page *page;
 
-	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask, 0);
  	if (page)
  		set_page_refcounted(page);
  	return page;
@@ -7856,80 +7931,10 @@ static bool __free_unaccepted(struct page *page)
 
  struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
  {
-	/*
-	 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
-	 * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
-	 * is not safe in arbitrary context.
-	 *
-	 * These two are the conditions for gfpflags_allow_spinning() being true.
-	 *
-	 * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
-	 * to warn. Also warn would trigger printk() which is unsafe from
-	 * various contexts. We cannot use printk_deferred_enter() to mitigate,
-	 * since the running context is unknown.
-	 *
-	 * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
-	 * is safe in any context. Also zeroing the page is mandatory for
-	 * BPF use cases.
-	 *
-	 * Though __GFP_NOMEMALLOC is not checked in the code path below,
-	 * specify it here to highlight that alloc_pages_nolock()
-	 * doesn't want to deplete reserves.
-	 */
-	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
-			| gfp_flags;
-	unsigned int alloc_flags = ALLOC_TRYLOCK;
-	struct alloc_context ac = { };
-	struct page *page;
-
-	VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
-	/*
-	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
-	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
-	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
-	 * mark the task as the owner of another rt_spin_lock which will
-	 * confuse PI logic, so return immediately if called from hard IRQ or
-	 * NMI.
-	 *
-	 * Note, irqs_disabled() case is ok. This function can be called
-	 * from raw_spin_lock_irqsave region.
-	 */
-	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
-		return NULL;
-
-	/* On UP, spin_trylock() always succeeds even when it is locked */
-	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
-		return NULL;
-
-	if (!pcp_allowed_order(order))
-		return NULL;
-
-	/* Bailout, since _deferred_grow_zone() needs to take a lock */
-	if (deferred_pages_enabled())
-		return NULL;
-
  	if (nid == NUMA_NO_NODE)
  		nid = numa_node_id();
 
-	prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac,
-			    &alloc_gfp, &alloc_flags);
-
-	/*
-	 * Best effort allocation from percpu free list.
-	 * If it's empty attempt to spin_trylock zone->lock.
-	 */
-	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-
-	/* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */
-
-	if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
-	    unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
-		__free_frozen_pages(page, order, FPI_TRYLOCK);
-		page = NULL;
-	}
-	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
-	kmsan_alloc_page(page, order, alloc_gfp);
-	return page;
+	return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_TRYLOCK);
  }
  /**
   * alloc_pages_nolock - opportunistic reentrant allocation from any context
mm/slub.c +2 -3
index a2bf3756ca7d0..b9fb66071bd07 100644
@@ -3275,7 +3275,7 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
  	else if (node == NUMA_NO_NODE)
  		page = alloc_frozen_pages(flags, order);
  	else
-		page = __alloc_frozen_pages(flags, order, node, NULL);
+		page = __alloc_frozen_pages(flags, order, node, NULL, 0);
 
  	if (!page)
  		return NULL;
@@ -5236,7 +5236,7 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
  	if (node == NUMA_NO_NODE)
  		page = alloc_frozen_pages_noprof(flags, order);
  	else
-		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
+		page = __alloc_frozen_pages_noprof(flags, order, node, NULL, 0);
 
  	if (page) {
  		ptr = page_address(page);
 
---

base-commit: 1111012ec6508a38a39f8d20c213c8c9cf3c96c0
change-id: 20260617-alloc-trylock-14ad37dab337

Best regards,
--
Brendan Jackman <[email protected]>

#10 lore raw
Quoted 49 lines · Hao Ge

Hi Brendan

Quoted 25 lines · Brendan Jackman

Currently the core allocator code is controlled by ALLOC_NOLOCK, but the main entry point function is significantly different from the normal __alloc_frozen_pages_nolock(), this is tiring when reading the code.

Plumb the ALLOC_NOLOCK control one layer up in the call stack: create an alloc_flags argument to __alloc_frozen_pages_nolock() (which is only exposed to mm/) and then turn the nolock variant into a thin wrapper that just sets that flag (as well as handling NUMA_NO_NODE, similar to how some of the wrappers in gfp.h do).

Rationale that this doesn't change anything:

1. Simple bits: A bunch of the nolock-specific handling is just moved to the new alloc_order_allowed(), alloc_trylock_allowed() and gfp_trylock.

2. __alloc_frozen_pages_noprof() has some extra logic that wasn't previously in the nolock variant:

a. Application of gfp_allowed_mask; this only affects early boot, and
only flags that affect the slowpath get changed here.

b. Application of current_gfp_context() - also only affects the
slowpath

Quoted 6 lines

3. The slowpath itself: this is now just explicitly skipped under !ALLOC_TRYLOCK.

Ulterior motive: adding an alloc_flags arg to the allocator's mm-internal entrypoint can later be used to do more allocation customisation without needing to create new GFP flags.

If so, I believe we could generalize this further.

Under the current logic, |__alloc_pages_slowpath| cannot access the alloc_flags
passed down from upper-level callers.
As I discussed in another thread, we can introduce a new alloc_flags to replace the |__GFP_NO_CODETAG| (|__GFP_NO_OBJ_EXT|) GFP flag. This newly added flag needs to be propagated along the entire call chain down to |prep_new_page|, which means |__alloc_pages_slowpath| also has to
handle this flag accordingly.

Quoted 5 lines

I'm wondering if we could introduce a caller_alloc_flags field within struct alloc_context
to handle alloc_flags that need to persist throughout the entire page allocation cycle, when such flags exist.
I'm sure others will have more appropriate solutions.

Yeah totally, this is exactly how I imagined this evolving.

#11 lore raw
In reply to Brendan Jackman
Quoted 8 lines · Brendan Jackman

+++ b/mm/page_alloc.c

mm/page_alloc.c +0 -3
@@ -5253,24 +5253,98 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
 	}
 }
 
-/*
- * This is the 'heart' of the zoned buddy allocator.
- */

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5259

Quoted 10 lines
mm/page_alloc.c +3 -6
-struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+				       unsigned int alloc_flags)
 {
-	struct page *page;
-	unsigned int alloc_flags = ALLOC_WMARK_LOW;
-	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-	struct alloc_context ac = { };
+

Spurious blank line?

Quoted 2 lines
mm/page_alloc.c +2 -0
+	if (alloc_flags & ALLOC_TRYLOCK)
+		return pcp_allowed_order(order);

[...]

Quoted 9 lines

+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *

Quoted 8 lines

+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.

It may be mandatory for BPF, but it seems wasteful for other uses.

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5320

Quoted 6 lines

+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */

mm/page_alloc.c +2 -0
+static const gfp_t gfp_trylock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+				__GFP_COMP;

I rather dislike this being turned into a file-scope variable, even a non-varying variable. Can't it remain inside a function?

Quoted 9 lines
mm/page_alloc.c +9 -0
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+	struct page *page;
+	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+	struct alloc_context ac = { };

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5340

Quoted 10 lines
mm/page_alloc.c +8 -0
+
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
 		return NULL;
 
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
+		return NULL;
+
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);

So the only GFP flag the user is allowed to specify is __GFP_ACCOUNT? That seems bogus; other flags would be reasonable including all the ones in gfp_trylock, as well as GFP_HIGHMEM, GFP_DMA, GFP_MOVABLE, GFP_HARDWALL.

#12 lore raw
Quoted 6 lines · Matthew Wilcox
Quoted 5 lines · Brendan Jackman

+++ b/mm/page_alloc.c

(quoted diff) +0 -0
@@ -5253,24 +5253,98 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
 	}
 }
 
Quoted 15 lines

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5259

Quoted 13 lines
mm/page_alloc.c +3 -9
-/*
- * This is the 'heart' of the zoned buddy allocator.
- */
-struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+				       unsigned int alloc_flags)
 {
-	struct page *page;
-	unsigned int alloc_flags = ALLOC_WMARK_LOW;
-	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-	struct alloc_context ac = { };
+

Spurious blank line?

Yep, thanks.

Quoted 21 lines
Quoted 2 lines
mm/page_alloc.c +2 -0
+	if (alloc_flags & ALLOC_TRYLOCK)
+		return pcp_allowed_order(order);

[...]

Quoted 9 lines

+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *

Quoted 8 lines

+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.

Quoted 1 line

It may be mandatory for BPF, but it seems wasteful for other uses.

True, don't see why we shouldn't push this out to the caller, I can do it as part of this series.

Quoted 9 lines

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5320

Quoted 6 lines

+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */

mm/page_alloc.c +2 -0
+static const gfp_t gfp_trylock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+				__GFP_COMP;

I rather dislike this being turned into a file-scope variable, even a non-varying variable. Can't it remain inside a function?

Um, we could put it into a function like void add_gfp_trylock(gfp_t *gfp) but that doesn't really reduce the scope in any meaninful way, right?

We could also squash it into what's currently called alloc_trylock_allowed but then it's a bit of a mush function, would it be called do_trlock_stuff?

Putting it directly into __alloc_frozen_pages_noprof() would make that function too big IMO, and its real estate would be dominated by trylock stuff.

It's definitely understandable to find the large variable scope yucky but IMO the real fix for that would be to break up page_alloc.c, which I don't really want to do in this series.

Quoted 14 lines
Quoted 14 lines
mm/page_alloc.c +12 -0
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+	struct page *page;
+	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+	struct alloc_context ac = { };
+
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
 		return NULL;
Quoted 9 lines

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5340

Quoted 5 lines
mm/page_alloc.c +5 -0
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
+		return NULL;
+
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);

So the only GFP flag the user is allowed to specify is __GFP_ACCOUNT? That seems bogus; other flags would be reasonable including all the ones in gfp_trylock, as well as GFP_HIGHMEM, GFP_DMA, GFP_MOVABLE, GFP_HARDWALL.

Definitely makes sense for the ones in gfp_trylock.

For the others, I'm not sure - this "nolock" functionality is a bit weird and sketchy, I suspect the reason for the WARN here is "let's make sure we have a proper think before we allow it to grow usecases that are meaningfully different from the other ones". I think I like that conservatism here, I would lean towards keeping it? Not a passionately held opinion though.

#13 lore raw
Quoted 21 lines · Brendan Jackman
Quoted 20 lines · Matthew Wilcox
Quoted 17 lines · Brendan Jackman

+/*
+ * GFP flags to set for ALLOC_TRYLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *
+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.

Quoted 4 lines
Quoted 1 line

It may be mandatory for BPF, but it seems wasteful for other uses.

True, don't see why we shouldn't push this out to the caller, I can do it as part of this series.

Wait, I wasn't paying attention to the bit about kmsan_alloc_page(). __GFP_ZERO is load bearing for the implementation because otherwise kmsan_alloc_page() does things that might not be safe in the current context.

Quoted 11 lines
Quoted 6 lines

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5340

Quoted 2 lines
mm/page_alloc.c +2 -0
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);

So the only GFP flag the user is allowed to specify is __GFP_ACCOUNT? That seems bogus; other flags would be reasonable including all the ones in gfp_trylock, as well as GFP_HIGHMEM, GFP_DMA, GFP_MOVABLE, GFP_HARDWALL.

Definitely makes sense for the ones in gfp_trylock.

For the others, I'm not sure - this "nolock" functionality is a bit weird and sketchy,

...

Quoted 5 lines

I suspect the reason for the WARN here is "let's make
sure we have a proper think before we allow it to grow usecases that are meaningfully different from the other ones". I think I like that conservatism here, I would lean towards keeping it? Not a passionately held opinion though.

I'd probably still lean towards pushing __GFP_ZERO to the caller though, and just WARN+fail if it isn't passed, to make it explicit?

#14 lore raw
In reply to Brendan Jackman
Quoted 15 lines · Brendan Jackman
Quoted 14 lines · Matthew Wilcox
Quoted 14 lines
mm/page_alloc.c +12 -0
+/*
+ * This is the 'heart' of the zoned buddy allocator.
+ */
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
+{
+	struct page *page;
+	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
+	struct alloc_context ac = { };
+
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~ALLOC_TRYLOCK))
 		return NULL;
Quoted 9 lines
Quoted 9 lines

mm/page_alloc.c void free_pages_bulk(struct page **page_array, unsigned long nr_pages) · line 5340

Quoted 5 lines
mm/page_alloc.c +5 -0
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
+		return NULL;
+
+	if (alloc_flags & ALLOC_TRYLOCK) {
+		VM_WARN_ON_ONCE(gfp & ~__GFP_ACCOUNT);

So the only GFP flag the user is allowed to specify is __GFP_ACCOUNT? That seems bogus; other flags would be reasonable including all the ones in gfp_trylock, as well as GFP_HIGHMEM, GFP_DMA, GFP_MOVABLE, GFP_HARDWALL.

We could start phasing out GFP_HIGHMEM and GFP_DMA somewhere :)

Quoted 5 lines

Definitely makes sense for the ones in gfp_trylock.

For the others, I'm not sure - this "nolock" functionality is a bit weird and sketchy, I suspect the reason for the WARN here is "let's make sure we have a proper think before we allow it to grow usecases that are

Yeah you were just keeping the same allowed flags that alloc_frozen_pages_nolock_noprof() currently does.

Quoted 3 lines

meaningfully different from the other ones". I think I like that conservatism here, I would lean towards keeping it? Not a passionately held opinion though.

Yeah if we expand it, I'd do it separately and in response to use cases appearing.
We want _nolock to be used only where really necessary, as its success is very limited, not to be (ab)used as a cheap way out of a wrongly designed driver or something. ("oh I need to allocate something here and can't spin, so let's just use this, solved").