mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Subsystems (4)
Hao Ge
reviewerPATCH v2 1/1 mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Pages allocated before page_ext is available have their codetag left uninitialized. Track these early PFNs and clear their codetag in clear_early_alloc_pfn_tag_refs() to avoid "alloc_tag was not set" warnings when they are freed later.
Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into early_pfn_node entries and the remainder is kept as a freelist for subsequent allocations.
The list nodes themselves are allocated via alloc_page(), which would trigger __pgalloc_tag_add() -> alloc_tag_add_early_pfn() -> alloc_early_pfn_node() and recurse indefinitely. Introduce __GFP_NO_CODETAG (reuses the %__GFP_NO_OBJ_EXT bit) and pass gfp_flags through pgalloc_tag_add() so that the early path can skip recording allocations that carry this flag.
---
v2:
- Use cmpxchg to atomically update early_pfn_pages, preventing page leak under concurrent allocation
- Pass gfp_flags through the full call chain and use gfpflags_allow_blocking()
to select GFP_KERNEL vs GFP_ATOMIC, avoiding unnecessary GFP_ATOMIC in process context
--- include/linux/alloc_tag.h | 22 +++++++- lib/alloc_tag.c | 102 ++++++++++++++++++++++++++------------ mm/page_alloc.c | 29 +++++++---- 3 files changed, 108 insertions(+), 45 deletions(-)
include/linux/alloc_tag.h +20 -2
@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),
@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }#endif /* Caller should verify both ref and tag to be valid */
lib/alloc_tag.c +69 -33
147 lines not on this page · raw message
mm/page_alloc.c +19 -10
@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;
@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation +* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; + +/* +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. +*/ +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */
@@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}
@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Signature · 1 line
2.25.1
Suren Baghdasaryan
maintainerQuoted 14 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into
early_pfn_nodeentries and the remainder is kept as a freelist for subsequent allocations.
Quoted 6 lines
The list nodes themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()->alloc_early_pfn_node()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.
Hi Hao,
Thanks for following up on this!
Quoted 43 lines
Signed-off-by Hao Ge <[email protected]>---
v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 +++++++- lib/alloc_tag.c | 102 ++++++++++++++++++++++++++------------ mm/page_alloc.c | 29 +++++++---- 3 files changed, 108 insertions(+), 45 deletions(-)include/linux/alloc_tag.h +17
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {
include/linux/alloc_tag.h static inline void alloc_tag_sub_check(union codetag_ref *ref) · line 188
Quoted 10 lines
include/linux/alloc_tag.h +3
-2WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }
If CONFIG_MEM_ALLOC_PROFILING_DEBUG=n why should we record early pfns?
Quoted 26 lines
include/linux/alloc_tag.h +0
-0#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +2
-12@@ -766,45 +766,75 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +struct early_pfn_node { +struct early_pfn_node *next;
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 779
Quoted 10 lines
lib/alloc_tag.c +7
-2+unsigned long pfn; +}; + +#define NODES_PER_PAGE (PAGE_SIZE / sizeof(struct early_pfn_node)) -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct early_pfn_node *early_pfn_list__initdata ; +static struct early_pfn_node *early_pfn_freelist__initdata ; +static struct page *early_pfn_pages__initdata ;
This early_pfn_node linked list seems overly complex. Why not just allocate a page and use page->lru to place it into a linked list? I think the code will end up much simpler.
Quoted 36 lines
lib/alloc_tag.c +29
-3-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static struct early_pfn_node *__init alloc_early_pfn_node (gfp_t gfp_flags) { -int old_idx, new_idx; +struct early_pfn_node *ep, *new; +struct page *page, *old_page; +gfp_t gfp =gfpflags_allow_blocking (gfp_flags) ?GFP_KERNEL :GFP_ATOMIC ; +int i; + +retry: + ep =READ_ONCE (early_pfn_freelist); +if (ep) { +struct early_pfn_node *next =READ_ONCE (ep->next); + +if (try_cmpxchg (&early_pfn_freelist, &ep, next)) +return ep; + goto retry; + } + + page =alloc_page (gfp | __GFP_NO_CODETAG | __GFP_ZERO); +if (!page) +return NULL ; + + new =page_address (page); +for (i =0 ; i <NODES_PER_PAGE -1 ; i++) + new[i].next = &new[i +1 ]; + new[NODES_PER_PAGE -1 ].next =NULL ; + +if (cmpxchg (&early_pfn_freelist,NULL , new +1 )) { +__free_page (page); + goto retry; + } do { - old_idx = atomic_read(&early_pfn_count);
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 815
Quoted 10 lines
lib/alloc_tag.c +3
-7-if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; - } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + old_page =READ_ONCE (early_pfn_pages); + page->private = (unsigned long )old_page; + }while (cmpxchg (&early_pfn_pages, old_page, page) != old_page);
I don't think this whole lockless schema is worth the complexity. alloc_early_pfn_node() is called only during early init and is called perhaps a few hundred times in total. Why not use a simple spinlock to synchronize this operation and be done with it?
Quoted 104 lines
lib/alloc_tag.c +28
-9+ +return new; +} + +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) +{ +struct early_pfn_node *ep =alloc_early_pfn_node (gfp_flags); - early_pfns[old_idx] = pfn; +if (!ep) +return ; + + ep->pfn = pfn; +do { + ep->next =READ_ONCE (early_pfn_list); + }while (!try_cmpxchg (&early_pfn_list, &ep->next, ep)); } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +844,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct early_pfn_node *ep; +struct page *page, *next;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,14 +860,13 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; +for (ep = early_pfn_list; ep; ep = ep->next) { -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); +if (pfn_valid (ep->pfn)) {union pgtag_ref_handle handle;union codetag_ref ref; + page =pfn_to_page (ep->pfn);if (get_page_tag_ref (page, &ref, &handle)) {/* * An early-allocated page could be freed and reallocated @@ -861,6 +891,12 @@ static void __init clear_early_alloc_pfn_tag_refs(void) } } + +for (page = early_pfn_pages; page; page = next) { + next = (struct page *)page->private; +clear_page_tag_ref (page); +__free_page (page); + } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline void __init clear_early_alloc_pfn_tag_refs (void ) {}mm/page_alloc.c +5
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1312
Quoted 10 lines
mm/page_alloc.c +10
-0+* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; + +/* +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. +*/ +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);
nit: This seems shorter and more readable:
if (unlikely(should_record_early_pfn(gfp_flags)))
alloc_tag_add_early_pfn(page_to_pfn(page),gfp_flags);
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Hao Ge
reviewerQuoted 14 lines · Suren Baghdasaryan
Quoted 13 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into
early_pfn_nodeentries and the remainder is kept as a freelist for subsequent allocations.
Quoted 6 lines
Quoted 6 lines
The list nodes themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()->alloc_early_pfn_node()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.
Hi Suren
Quoted 2 lines
Hi Hao,
Thanks for following up on this!
Happy to help further develop this feature.
Feel free to reach out if there's anything else I can do.
Quoted 47 lines
Quoted 47 lines
Signed-off-by Hao Ge <[email protected]>---
v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 +++++++- lib/alloc_tag.c | 102 ++++++++++++++++++++++++++------------ mm/page_alloc.c | 29 +++++++---- 3 files changed, 108 insertions(+), 45 deletions(-)include/linux/alloc_tag.h +18
-1@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);
Quoted 8 lines
include/linux/alloc_tag.h static inline void alloc_tag_sub_check(union codetag_ref *ref) · line 188
Quoted 6 lines
include/linux/alloc_tag.h +2
-1#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }If
CONFIG_MEM_ALLOC_PROFILING_DEBUG=nwhy should we record early pfns?
Good point! I'll address this in the next patch.
Quoted 31 lines
Quoted 31 lines
include/linux/alloc_tag.h +0
-0#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +6
-12@@ -766,45 +766,75 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +struct early_pfn_node { +struct early_pfn_node *next; +unsigned long pfn; +}; + +#define NODES_PER_PAGE (PAGE_SIZE / sizeof(struct early_pfn_node))
Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 779
Quoted 5 lines
lib/alloc_tag.c +3
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct early_pfn_node *early_pfn_list__initdata ; +static struct early_pfn_node *early_pfn_freelist__initdata ; +static struct page *early_pfn_pages__initdata ;This
early_pfn_nodelinked list seems overly complex. Why not just allocate a page and usepage->lruto place it into a linked list? I think the code will end up much simpler.
Ah, yes! You mentioned this before, but I misunderstood your point.
I apologize for the confusion. I'll optimize this in the next revision.
Quoted 13 lines
Quoted 13 lines
lib/alloc_tag.c +9
-2-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static struct early_pfn_node *__init alloc_early_pfn_node (gfp_t gfp_flags) { -int old_idx, new_idx; +struct early_pfn_node *ep, *new; +struct page *page, *old_page; +gfp_t gfp =gfpflags_allow_blocking (gfp_flags) ?GFP_KERNEL :GFP_ATOMIC ; +int i; + +retry: + ep =READ_ONCE (early_pfn_freelist); +if (ep) {
Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 798
Quoted 8 lines
lib/alloc_tag.c +8
-0+struct early_pfn_node *next =READ_ONCE (ep->next); + +if (try_cmpxchg (&early_pfn_freelist, &ep, next)) +return ep; + goto retry; + } + + page =alloc_page (gfp | __GFP_NO_CODETAG | __GFP_ZERO);
One more question: since this is called in an RCU context, should we use GFP_ATOMIC by default?
Also, should we remove GFP_KSWAPD_RECLAIM here? I'm not entirely sure, but I recall Sashiko mentioned a warning about this before:
---
page = alloc_page(GFP_ATOMIC | __GFP_NO_CODETAG | __GFP_ZERO);
Sashiko's concerns:
Can this lead to a deadlock by introducing lock recursion? alloc_early_pfn_node() is invoked as a post-allocation hook for early boot pages via pgalloc_tag_add(). GFP_ATOMIC includes __GFP_KSWAPD_RECLAIM, which triggers wakeup_kswapd() and acquires scheduler locks.
If the original allocation was made under scheduler locks and intentionally stripped __GFP_KSWAPD_RECLAIM to prevent recursion, does this hardcoded GFP_ATOMIC force it back on? Should the hook inherit or constrain its flags based on the caller's gfp_flags instead?
---
Even though this only happens during early boot, should we handle it more safely?
Quoted 13 lines
Quoted 13 lines
lib/alloc_tag.c +12
-0+if (!page) +return NULL ; + + new =page_address (page); +for (i =0 ; i <NODES_PER_PAGE -1 ; i++) + new[i].next = &new[i +1 ]; + new[NODES_PER_PAGE -1 ].next =NULL ; + +if (cmpxchg (&early_pfn_freelist,NULL , new +1 )) { +__free_page (page); + goto retry; + }
Quoted 16 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 815
Quoted 12 lines
lib/alloc_tag.c +3
-8do { - old_idx =atomic_read (&early_pfn_count); -if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; - } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + old_page =READ_ONCE (early_pfn_pages); + page->private = (unsigned long )old_page; + }while (cmpxchg (&early_pfn_pages, old_page, page) != old_page);I don't think this whole lockless schema is worth the complexity.
alloc_early_pfn_node()is called only during early init and is called perhaps a few hundred times in total. Why not use a simple spinlock to synchronize this operation and be done with it?
I initially used a simple spinlock, but Sashiko raised a good point in his review:
https://sashiko.dev/#/patchset/20260319083153.2488005-1-hao.ge%40linux.dev
Since alloc_early_pfn_node() is called during early init from an unknown context, in the early page allocation path,
a lockless approach is safer. However, you're right that if we use page->lru as a linked list (which you suggested earlier),
the code becomes much simpler. I plan to simplify the code and continue using the lockless approach in the next version.
Quoted 116 lines
Quoted 104 lines
lib/alloc_tag.c +28
-9+ +return new; +} + +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) +{ +struct early_pfn_node *ep =alloc_early_pfn_node (gfp_flags); - early_pfns[old_idx] = pfn; +if (!ep) +return ; + + ep->pfn = pfn; +do { + ep->next =READ_ONCE (early_pfn_list); + }while (!try_cmpxchg (&early_pfn_list, &ep->next, ep)); } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +844,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct early_pfn_node *ep; +struct page *page, *next;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,14 +860,13 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; +for (ep = early_pfn_list; ep; ep = ep->next) { -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); +if (pfn_valid (ep->pfn)) {union pgtag_ref_handle handle;union codetag_ref ref; + page =pfn_to_page (ep->pfn);if (get_page_tag_ref (page, &ref, &handle)) {/* * An early-allocated page could be freed and reallocated @@ -861,6 +891,12 @@ static void __init clear_early_alloc_pfn_tag_refs(void) } } + +for (page = early_pfn_pages; page; page = next) { + next = (struct page *)page->private; +clear_page_tag_ref (page); +__free_page (page); + } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline void __init clear_early_alloc_pfn_tag_refs (void ) {}mm/page_alloc.c +5
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1312
Quoted 10 lines
mm/page_alloc.c +10
-0+* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; + +/* +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. +*/ +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);nit: This seems shorter and more readable:
Quoted 3 lines
if (unlikely(should_record_early_pfn(gfp_flags)))
alloc_tag_add_early_pfn(page_to_pfn(page),gfp_flags);
OK, will improve it in the next version.
Thanks for taking the time to review and provide these suggestions!
Thanks
Best Regards
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Suren Baghdasaryan
maintainerQuoted 148 lines · Hao Ge
Quoted 14 lines · Suren Baghdasaryan
Quoted 13 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into
early_pfn_nodeentries and the remainder is kept as a freelist for subsequent allocations.Quoted 6 lines
Quoted 6 lines
The list nodes themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()->alloc_early_pfn_node()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Hi Suren
Quoted 2 lines
Hi Hao,
Thanks for following up on this!Happy to help further develop this feature.
Feel free to reach out if there's anything else I can do.
Quoted 47 lines
Quoted 47 lines
Signed-off-by Hao Ge <[email protected]>---
v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 +++++++- lib/alloc_tag.c | 102 ++++++++++++++++++++++++++------------ mm/page_alloc.c | 29 +++++++---- 3 files changed, 108 insertions(+), 45 deletions(-)include/linux/alloc_tag.h +18
-1@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);Quoted 8 lines
include/linux/alloc_tag.h static inline void alloc_tag_sub_check(union codetag_ref *ref) · line 188
Quoted 6 lines
include/linux/alloc_tag.h +2
-1#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }If
CONFIG_MEM_ALLOC_PROFILING_DEBUG=nwhy should we record early pfns?Good point! I'll address this in the next patch.
Quoted 31 lines
Quoted 31 lines
include/linux/alloc_tag.h +0
-0#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +6
-12@@ -766,45 +766,75 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +struct early_pfn_node { +struct early_pfn_node *next; +unsigned long pfn; +}; + +#define NODES_PER_PAGE (PAGE_SIZE / sizeof(struct early_pfn_node))Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 779
Quoted 5 lines
lib/alloc_tag.c +3
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct early_pfn_node *early_pfn_list__initdata ; +static struct early_pfn_node *early_pfn_freelist__initdata ; +static struct page *early_pfn_pages__initdata ;This
early_pfn_nodelinked list seems overly complex. Why not just allocate a page and usepage->lruto place it into a linked list? I think the code will end up much simpler.Ah, yes! You mentioned this before, but I misunderstood your point.
I apologize for the confusion. I'll optimize this in the next revision.
Quoted 16 lines
Quoted 16 lines
lib/alloc_tag.c +12
-2-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static struct early_pfn_node *__init alloc_early_pfn_node (gfp_t gfp_flags) { -int old_idx, new_idx; +struct early_pfn_node *ep, *new; +struct page *page, *old_page; +gfp_t gfp =gfpflags_allow_blocking (gfp_flags) ?GFP_KERNEL :GFP_ATOMIC ; +int i; + +retry: + ep =READ_ONCE (early_pfn_freelist); +if (ep) { +struct early_pfn_node *next =READ_ONCE (ep->next); + +if (try_cmpxchg (&early_pfn_freelist, &ep, next))
Quoted 8 lines
Quoted 5 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 798
Quoted 5 lines
lib/alloc_tag.c +5
-0+return ep; + goto retry; + } + + page =alloc_page (gfp | __GFP_NO_CODETAG | __GFP_ZERO);One more question: since this is called in an RCU context, should we use
GFP_ATOMICby default?
You mean it might be called in RCU context, right? If so then I think we should not use GFP_ATOMIC when GFP_KERNEL can be used.
Quoted 19 lines
Also, should we remove
GFP_KSWAPD_RECLAIMhere? I'm not entirely sure, but I recall Sashiko mentioned a warning about this before:---
page =
alloc_page(GFP_ATOMIC | __GFP_NO_CODETAG | __GFP_ZERO);Sashiko's concerns:
Can this lead to a deadlock by introducing lock recursion?
alloc_early_pfn_node()is invoked as a post-allocation hook for early boot pages viapgalloc_tag_add().GFP_ATOMICincludes__GFP_KSWAPD_RECLAIM, which triggerswakeup_kswapd()and acquires scheduler locks.
If the original allocation was made under scheduler locks and intentionally stripped__GFP_KSWAPD_RECLAIMto prevent recursion, does this hardcodedGFP_ATOMICforce it back on? Should the hook inherit or constrain its flags based on the caller'sgfp_flagsinstead?
Quoted 4 lines
---
Even though this only happens during early boot, should we handle it more safely?
That seems reasonable. Any reason you are not doing simply: page = alloc_page(gfp_flags | __GFP_NO_CODETAG | __GFP_ZERO); ? IOW why aren't you simply inheriting the flags?
Quoted 36 lines
Quoted 14 lines
Quoted 14 lines
lib/alloc_tag.c +12
-0+if (!page) +return NULL ; + + new =page_address (page); +for (i =0 ; i <NODES_PER_PAGE -1 ; i++) + new[i].next = &new[i +1 ]; + new[NODES_PER_PAGE -1 ].next =NULL ; + +if (cmpxchg (&early_pfn_freelist,NULL , new +1 )) { +__free_page (page); + goto retry; + }Quoted 16 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 815
Quoted 12 lines
lib/alloc_tag.c +3
-8do { - old_idx =atomic_read (&early_pfn_count); -if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; - } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + old_page =READ_ONCE (early_pfn_pages); + page->private = (unsigned long )old_page; + }while (cmpxchg (&early_pfn_pages, old_page, page) != old_page);I don't think this whole lockless schema is worth the complexity.
alloc_early_pfn_node()is called only during early init and is called perhaps a few hundred times in total. Why not use a simple spinlock to synchronize this operation and be done with it?I initially used a simple spinlock, but Sashiko raised a good point in his review:
https://sashiko.dev/#/patchset/20260319083153.2488005-1-hao.ge%40linux.dev
Quoted 8 lines
Since
alloc_early_pfn_node()is called during early init from an unknown context, in the early page allocation path,a lockless approach is safer. However, you're right that if we use
page->lruas a linked list (which you suggested earlier),the code becomes much simpler. I plan to simplify the code and continue using the lockless approach in the next version.
Ok, let's see the result and decide then. Allocating a page from NMI context sounds extreme to me. I would expect NMI context to use only preallocated memory.
Quoted 118 lines
Quoted 118 lines
Quoted 106 lines
lib/alloc_tag.c +28
-9+ +return new; +} + +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) +{ +struct early_pfn_node *ep =alloc_early_pfn_node (gfp_flags); - early_pfns[old_idx] = pfn; +if (!ep) +return ; + + ep->pfn = pfn; +do { + ep->next =READ_ONCE (early_pfn_list); + }while (!try_cmpxchg (&early_pfn_list, &ep->next, ep)); } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +844,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct early_pfn_node *ep; +struct page *page, *next;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,14 +860,13 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; +for (ep = early_pfn_list; ep; ep = ep->next) { -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); +if (pfn_valid (ep->pfn)) {union pgtag_ref_handle handle;union codetag_ref ref; + page =pfn_to_page (ep->pfn);if (get_page_tag_ref (page, &ref, &handle)) {/* * An early-allocated page could be freed and reallocated @@ -861,6 +891,12 @@ static void __init clear_early_alloc_pfn_tag_refs(void) } } + +for (page = early_pfn_pages; page; page = next) { + next = (struct page *)page->private; +clear_page_tag_ref (page); +__free_page (page); + } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline void __init clear_early_alloc_pfn_tag_refs (void ) {}mm/page_alloc.c +5
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1312
Quoted 10 lines
mm/page_alloc.c +10
-0+* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; + +/* +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. +*/ +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);nit: This seems shorter and more readable:
Quoted 5 lines
Quoted 3 lines
if (unlikely(should_record_early_pfn(gfp_flags)))
alloc_tag_add_early_pfn(page_to_pfn(page),gfp_flags);OK, will improve it in the next version.
Thanks,
Suren.
Quoted 44 lines
Thanks for taking the time to review and provide these suggestions!
Thanks
Best Regards
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Hao Ge
reviewerQuoted 150 lines · Suren Baghdasaryan
Quoted 141 lines · Hao Ge
Quoted 14 lines · Suren Baghdasaryan
Quoted 13 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into
early_pfn_nodeentries and the remainder is kept as a freelist for subsequent allocations.Quoted 6 lines
Quoted 6 lines
The list nodes themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()->alloc_early_pfn_node()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Hi Suren
Quoted 2 lines
Hi Hao,
Thanks for following up on this!Happy to help further develop this feature.
Feel free to reach out if there's anything else I can do.
Quoted 45 lines
Quoted 45 lines
Signed-off-by Hao Ge <[email protected]>---
v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 +++++++- lib/alloc_tag.c | 102 ++++++++++++++++++++++++++------------ mm/page_alloc.c | 29 +++++++---- 3 files changed, 108 insertions(+), 45 deletions(-)include/linux/alloc_tag.h +17
-1@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn);Quoted 8 lines
include/linux/alloc_tag.h static inline void alloc_tag_sub_check(union codetag_ref *ref) · line 188
Quoted 7 lines
include/linux/alloc_tag.h +3
-1+void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }If
CONFIG_MEM_ALLOC_PROFILING_DEBUG=nwhy should we record early pfns?Good point! I'll address this in the next patch.
Quoted 31 lines
Quoted 31 lines
include/linux/alloc_tag.h +0
-0#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +6
-12@@ -766,45 +766,75 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +struct early_pfn_node { +struct early_pfn_node *next; +unsigned long pfn; +}; + +#define NODES_PER_PAGE (PAGE_SIZE / sizeof(struct early_pfn_node))Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 779
Quoted 5 lines
lib/alloc_tag.c +3
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct early_pfn_node *early_pfn_list__initdata ; +static struct early_pfn_node *early_pfn_freelist__initdata ; +static struct page *early_pfn_pages__initdata ;This
early_pfn_nodelinked list seems overly complex. Why not just allocate a page and usepage->lruto place it into a linked list? I think the code will end up much simpler.Ah, yes! You mentioned this before, but I misunderstood your point.
I apologize for the confusion. I'll optimize this in the next revision.
Quoted 14 lines
Quoted 14 lines
lib/alloc_tag.c +11
-2-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static struct early_pfn_node *__init alloc_early_pfn_node (gfp_t gfp_flags) { -int old_idx, new_idx; +struct early_pfn_node *ep, *new; +struct page *page, *old_page; +gfp_t gfp =gfpflags_allow_blocking (gfp_flags) ?GFP_KERNEL :GFP_ATOMIC ; +int i; + +retry: + ep =READ_ONCE (early_pfn_freelist); +if (ep) { +struct early_pfn_node *next =READ_ONCE (ep->next); +Quoted 8 lines
Quoted 6 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 798
Quoted 6 lines
lib/alloc_tag.c +6
-0+if (try_cmpxchg (&early_pfn_freelist, &ep, next)) +return ep; + goto retry; + } + + page =alloc_page (gfp | __GFP_NO_CODETAG | __GFP_ZERO);One more question: since this is called in an RCU context, should we use
GFP_ATOMICby default?
Quoted 2 lines
You mean it might be called in RCU context, right? If so then I think we should not use
GFP_ATOMICwhenGFP_KERNELcan be used.
Hi Suren
__alloc_tag_add_early_pfn() is invoked under rcu_read_lock(). The full call chain is:
post_alloc_hook()
-> pgalloc_tag_add()
-> __pgalloc_tag_add()
-> alloc_tag_add_early_pfn(pfn, gfp_flags)
rcu_read_lock();
alloc_tag_add(pfn, gfp_flags)
-> __alloc_tag_add_early_pfn()
-> alloc_page(gfp) // lib/alloc_tag.c:783
rcu_read_unlock();
Quoted 19 lines
Quoted 19 lines
Also, should we remove
GFP_KSWAPD_RECLAIMhere? I'm not entirely sure, but I recall Sashiko mentioned a warning about this before:---
page =
alloc_page(GFP_ATOMIC | __GFP_NO_CODETAG | __GFP_ZERO);Sashiko's concerns:
Can this lead to a deadlock by introducing lock recursion?
alloc_early_pfn_node()is invoked as a post-allocation hook for early boot pages viapgalloc_tag_add().GFP_ATOMICincludes__GFP_KSWAPD_RECLAIM, which triggerswakeup_kswapd()and acquires scheduler locks.
If the original allocation was made under scheduler locks and intentionally stripped__GFP_KSWAPD_RECLAIMto prevent recursion, does this hardcodedGFP_ATOMICforce it back on? Should the hook inherit or constrain its flags based on the caller'sgfp_flagsinstead?
Quoted 7 lines
Quoted 4 lines
---
Even though this only happens during early boot, should we handle it more safely?
That seems reasonable. Any reason you are not doing simply: page =
alloc_page(gfp_flags | __GFP_NO_CODETAG | __GFP_ZERO); ? IOW why aren't you simply inheriting the flags?
As I described above, __alloc_tag_add_early_pfn() is invoked under rcu_read_lock().
We cannot simply pass through gfp_flags, since it may cause sleep,
That is the reason why I chose to use GFP_ATOMIC initially.
Thanks
Best Ragards
Hao
Quoted 213 lines
Quoted 34 lines
Quoted 13 lines
Quoted 13 lines
lib/alloc_tag.c +12
-0+if (!page) +return NULL ; + + new =page_address (page); +for (i =0 ; i <NODES_PER_PAGE -1 ; i++) + new[i].next = &new[i +1 ]; + new[NODES_PER_PAGE -1 ].next =NULL ; + +if (cmpxchg (&early_pfn_freelist,NULL , new +1 )) { +__free_page (page); + goto retry; + }Quoted 16 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 815
Quoted 12 lines
lib/alloc_tag.c +3
-8do { - old_idx =atomic_read (&early_pfn_count); -if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; - } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + old_page =READ_ONCE (early_pfn_pages); + page->private = (unsigned long )old_page; + }while (cmpxchg (&early_pfn_pages, old_page, page) != old_page);I don't think this whole lockless schema is worth the complexity.
alloc_early_pfn_node()is called only during early init and is called perhaps a few hundred times in total. Why not use a simple spinlock to synchronize this operation and be done with it?I initially used a simple spinlock, but Sashiko raised a good point in his review:
https://sashiko.dev/#/patchset/20260319083153.2488005-1-hao.ge%40linux.dev
Quoted 8 lines
Since
alloc_early_pfn_node()is called during early init from an unknown context, in the early page allocation path,a lockless approach is safer. However, you're right that if we use
page->lruas a linked list (which you suggested earlier),the code becomes much simpler. I plan to simplify the code and continue using the lockless approach in the next version.
Ok, let's see the result and decide then. Allocating a page from NMI context sounds extreme to me. I would expect NMI context to use only preallocated memory.
Quoted 117 lines
Quoted 117 lines
Quoted 105 lines
lib/alloc_tag.c +28
-9+ +return new; +} + +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) +{ +struct early_pfn_node *ep =alloc_early_pfn_node (gfp_flags); - early_pfns[old_idx] = pfn; +if (!ep) +return ; + + ep->pfn = pfn; +do { + ep->next =READ_ONCE (early_pfn_list); + }while (!try_cmpxchg (&early_pfn_list, &ep->next, ep)); } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +844,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct early_pfn_node *ep; +struct page *page, *next;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,14 +860,13 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; +for (ep = early_pfn_list; ep; ep = ep->next) { -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); +if (pfn_valid (ep->pfn)) {union pgtag_ref_handle handle;union codetag_ref ref; + page =pfn_to_page (ep->pfn);if (get_page_tag_ref (page, &ref, &handle)) {/* * An early-allocated page could be freed and reallocated @@ -861,6 +891,12 @@ static void __init clear_early_alloc_pfn_tag_refs(void) } } + +for (page = early_pfn_pages; page; page = next) { + next = (struct page *)page->private; +clear_page_tag_ref (page); +__free_page (page); + } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline void __init clear_early_alloc_pfn_tag_refs (void ) {}mm/page_alloc.c +5
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1312
Quoted 10 lines
mm/page_alloc.c +10
-0+* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; + +/* +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. +*/ +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);nit: This seems shorter and more readable:
Quoted 4 lines
Quoted 3 lines
if (unlikely(should_record_early_pfn(gfp_flags)))
alloc_tag_add_early_pfn(page_to_pfn(page),gfp_flags);OK, will improve it in the next version.
Thanks,
Suren.Quoted 43 lines
Thanks for taking the time to review and provide these suggestions!
Thanks
Best Regards
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1