mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Subsystems (4)
Hao Ge
reviewerPATCH v3 1/1 mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Pages allocated before page_ext is available have their codetag left uninitialized. Track these early PFNs and clear their codetag in clear_early_alloc_pfn_tag_refs() to avoid "alloc_tag was not set" warnings when they are freed later.
Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses page->lru to link to the next page, with page->private tracking the number of used slots.
The list pages themselves are allocated via alloc_page(), which would trigger __pgalloc_tag_add() -> alloc_tag_add_early_pfn() and recurse indefinitely. Introduce __GFP_NO_CODETAG (reuses the %__GFP_NO_OBJ_EXT bit) and pass gfp_flags through pgalloc_tag_add() so that the early path can skip recording allocations that carry this flag.
---
v3:
- Simplify linked list: use page->lru for chaining and page->private as slot counter, removing the early_pfn_node struct and freelist (suggested by Suren Baghdasaryan)
- Pass gfp_flags through alloc_tag_add_early_pfn() but strip __GFP_DIRECT_RECLAIM instead of selecting GFP_KERNEL/GFP_ATOMIC, because __alloc_tag_add_early_pfn() is invoked under rcu_read_lock().
v2:
- Use cmpxchg to atomically update early_pfn_pages, preventing page leak under concurrent allocation
- Pass gfp_flags through the full call chain and use gfpflags_allow_blocking()
to select GFP_KERNEL vs GFP_ATOMIC, avoiding unnecessary GFP_ATOMIC in process context
--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)
include/linux/alloc_tag.h +20 -2
@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),
@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */
lib/alloc_tag.c +64 -54
159 lines not on this page · raw message
mm/page_alloc.c +13 -10
@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;
@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags)) +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */
@@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}
@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Signature · 1 line
2.25.1
Suren Baghdasaryan
maintainerQuoted 49 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning.
nit: Callers must... sounds like someone who uses __alloc_tag_add_early_pfn() has to do something extra, however clear_page_tag_ref() is done by clear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:
Codetags of the pages allocated with __GFP_NO_CODETAG should be cleared (via clear_page_tag_ref()) before freeing the pages to prevent alloc_tag_sub_check() from triggering a warning.
Quoted 41 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +0
-8@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user.
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 10 lines
lib/alloc_tag.c +2
-6-* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long)) -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;
So, instead of using early_pfn_current as the head page and then page->lru.next directly, I would suggest having "struct llist_head early_pfn_head;" as the list head and either using page->pcp_llist to link the pages or even better, adding a "struct llist_node llist;" into the union containing "struct list_head lru;". Note that llist_add() uses try_cmpxchg(), so the link addition is atomic. And you should probably use folios rather than pages to follow the new trends.
Quoted 28 lines
lib/alloc_tag.c +17
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new->lru.next = (struct list_head *)page; + if (cmpxchg(&early_pfn_current, page, new) != page) { + __free_page(new); + continue; + }
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 10 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;
nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx)
break;Quoted 51 lines
lib/alloc_tag.c +7
-19+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY.
lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 10 lines
lib/alloc_tag.c +3
-7-* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic -* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];
This page_address() can be done in the outer loop since the page does not change inside the inner loop.
Quoted 63 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +1
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page));
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 10 lines
mm/page_alloc.c +8
-0+if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags)) +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);
Any reason for changing the order of alloc_tag_add_early_pfn() and alloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also call should_record_early_pfn() from inside alloc_tag_add_early_pfn(), which would keep both should_record_early_pfn() and __GFP_NO_CODETAG definitions local to alloc_tag.c.
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Suren Baghdasaryan
maintainerQuoted 119 lines · Suren Baghdasaryan
Quoted 49 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. nit: Callers must... sounds like someone who uses
__alloc_tag_add_early_pfn()has to do something extra, howeverclear_page_tag_ref()is done byclear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:Codetags of the pages allocated with
__GFP_NO_CODETAGshould be cleared (viaclear_page_tag_ref()) before freeing the pages to preventalloc_tag_sub_check()from triggering a warning.Quoted 48 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +1
-12@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
Quoted 11 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 3 lines
lib/alloc_tag.c +1
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;So, instead of using
early_pfn_currentas the head page and thenpage->lru.nextdirectly, I would suggest having "structllist_headearly_pfn_head;" as the list head and either usingpage->pcp_llistto link the pages or even better, adding a "structllist_nodellist;" into the union containing "structlist_headlru;". Note thatllist_add()usestry_cmpxchg(), so the link addition is atomic. And you should probably use folios rather than pages to follow the new trends.
Please ignore that last suggestion about using folios. I checked with Matthew and he says to use folios when needed, not whenever we can. So, using pages here seems fine.
Quoted 227 lines
Quoted 29 lines
lib/alloc_tag.c +17
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new->lru.next = (struct list_head *)page; + if (cmpxchg(&early_pfn_current, page, new) != page) { + __free_page(new); + continue; + }lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 10 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx) break;Quoted 51 lines
lib/alloc_tag.c +7
-19+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 10 lines
lib/alloc_tag.c +3
-7-* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic -* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];This
page_address()can be done in the outer loop since the page does not change inside the inner loop.Quoted 63 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +1
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page));mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 10 lines
mm/page_alloc.c +8
-0+if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags)) +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);Any reason for changing the order of
alloc_tag_add_early_pfn()andalloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also callshould_record_early_pfn()from insidealloc_tag_add_early_pfn(), which would keep bothshould_record_early_pfn()and__GFP_NO_CODETAGdefinitions local toalloc_tag.c.Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Hao Ge
reviewerHi Suren
Thanks a lot for your review and suggestions.
Quoted 64 lines · Suren Baghdasaryan
Quoted 48 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. nit: Callers must... sounds like someone who uses
__alloc_tag_add_early_pfn()has to do something extra, howeverclear_page_tag_ref()is done byclear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:
Quoted 3 lines
Codetags of the pages allocated with
__GFP_NO_CODETAGshould be cleared (viaclear_page_tag_ref()) before freeing the pages to preventalloc_tag_sub_check()from triggering a warning.
That's reasonable.
I will adjust the wording in v2.
Quoted 48 lines
Quoted 48 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +1
-12@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
Quoted 9 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 3 lines
lib/alloc_tag.c +1
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;So, instead of using
early_pfn_currentas the head page and thenpage->lru.nextdirectly, I would suggest having "structllist_headearly_pfn_head;" as the list head and either usingpage->pcp_llistto link the pages or even better, adding a "structllist_nodellist;" into the union containing "structlist_headlru;". Note thatllist_add()usestry_cmpxchg(), so the link addition is atomic.
Adding a struct llist_node into the page union only for the CONFIG_MEM_ALLOC_PROFILING_DEBUG feature,
which may not be quite appropriate from a design perspective.
Quoted 21 lines
And you should probably use folios rather than pages to follow the new trends.
Quoted 19 lines
lib/alloc_tag.c +9
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) {
Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 793
Quoted 8 lines
lib/alloc_tag.c +8
-0+pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new->lru.next = (struct list_head *)page; +if (cmpxchg (&early_pfn_current, page, new) != page) { +__free_page (new); +continue ; + }
I also considered using llist_head initially. like this
static LLIST_HEAD(early_pfn_pages);
static void __init __alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {
struct page *page;
unsigned long idx;
do {
page = llist_first(&early_pfn_pages);
if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; struct page *new = alloc_page(gfp | __GFP_NO_CODETAG);
if (!new) {
pr_warn_once("early PFN tracking page allocation failed\n"); return;
}
new->private = 0;
llist_add(&new->pcp_llist, &early_pfn_pages);
page = new;
}
idx = READ_ONCE(page->private);
if (idx >= EARLY_PFNS_PER_PAGE)
continue;
if (cmpxchg(&page->private, idx, idx + 1) == idx)
break;
} while (1);
((unsigned long *)page_address(page))[idx] = pfn; }
The problem is that llist_add() retries internally until it succeeds.
When multiple CPUs concurrently see that the head page is NULL or full ,
they each allocate a new tracking page and all insertions succeed:
CPU0: head page A is full or NULL, allocates B, llist_add(B) → head=B→A
CPU1: head page A is full or NULL, allocates C, llist_add(C) → head=C→B→A
CPU0: claims slot 0 in B
CPU1: claims slot 0 in C
Now the head is C. Once C fills up, a new page D is allocated and becomes the new head. But B still has ~511 unused slots and will never
be the head again — we only ever look at llist_first(). Those slots are permanently wasted.
That's why I used struct page *early_pfn_current with direct cmpxchg — the loser frees its page and retries, avoiding the waste.
That said, I may be missing something. Is there a way to use llist that avoids this? I'd appreciate your thoughts.
Quoted 10 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 10 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;
Quoted 4 lines
nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx) break;
will fix in the next version.
Quoted 56 lines
Quoted 56 lines
lib/alloc_tag.c +7
-23+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic
Quoted 8 lines
lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 6 lines
lib/alloc_tag.c +3
-3-* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];This
page_address()can be done in the outer loop since the page does not change inside the inner loop.
will fix in the next version.
Quoted 72 lines
Quoted 72 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +8
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags))
Quoted 9 lines
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 1 line
mm/page_alloc.c +1
-0+alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);Any reason for changing the order of
alloc_tag_add_early_pfn()andalloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also callshould_record_early_pfn()from insidealloc_tag_add_early_pfn(), which would keep bothshould_record_early_pfn()and__GFP_NO_CODETAGdefinitions local toalloc_tag.c.
will fix in the next version.
Thanks
Best regards.
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Suren Baghdasaryan
maintainerQuoted 228 lines · Hao Ge
Hi Suren
Thanks a lot for your review and suggestions.
Quoted 64 lines · Suren Baghdasaryan
Quoted 48 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. nit: Callers must... sounds like someone who uses
__alloc_tag_add_early_pfn()has to do something extra, howeverclear_page_tag_ref()is done byclear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:Quoted 3 lines
Codetags of the pages allocated with
__GFP_NO_CODETAGshould be cleared (viaclear_page_tag_ref()) before freeing the pages to preventalloc_tag_sub_check()from triggering a warning.That's reasonable.
I will adjust the wording in v2.
Quoted 48 lines
Quoted 48 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +1
-12@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))Quoted 9 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 3 lines
lib/alloc_tag.c +1
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;So, instead of using
early_pfn_currentas the head page and thenpage->lru.nextdirectly, I would suggest having "structllist_headearly_pfn_head;" as the list head and either usingpage->pcp_llistto link the pages or even better, adding a "structllist_nodellist;" into the union containing "structlist_headlru;". Note thatllist_add()usestry_cmpxchg(), so the link addition is atomic.Adding a struct
llist_nodeinto the page union only for theCONFIG_MEM_ALLOC_PROFILING_DEBUGfeature,which may not be quite appropriate from a design perspective.
Quoted 21 lines
And you should probably use folios rather than pages to follow the new trends.
Quoted 19 lines
lib/alloc_tag.c +9
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) {Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 793
Quoted 8 lines
lib/alloc_tag.c +8
-0+pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new->lru.next = (struct list_head *)page; +if (cmpxchg (&early_pfn_current, page, new) != page) { +__free_page (new); +continue ; + }I also considered using
llist_headinitially. like thisstatic
LLIST_HEAD(early_pfn_pages);static void
__init__alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags){ struct page *page; unsigned long idx; do { page = llist_first(&early_pfn_pages); if (!page || READ_ONCE(page->private) >=
EARLY_PFNS_PER_PAGE) {
gfp_tgfp =gfp_flags& ~__GFP_DIRECT_RECLAIM; struct page *new = alloc_page(gfp |__GFP_NO_CODETAG);if (!new) {
pr_warn_once("early PFN tracking pageallocation failed\n"); return; } new->private = 0; llist_add(&new->pcp_llist, &early_pfn_pages); page = new; } idx = READ_ONCE(page->private); if (idx >= EARLY_PFNS_PER_PAGE) continue; if (cmpxchg(&page->private, idx, idx + 1) == idx) break; } while (1); ((unsigned long *)page_address(page))[idx] = pfn; }The problem is that
llist_add()retries internally until it succeeds.When multiple CPUs concurrently see that the head page is NULL or full ,
they each allocate a new tracking page and all insertions succeed:
CPU0: head page A is full or NULL, allocates B,
llist_add(B)→ head=B→ACPU1: head page A is full or NULL, allocates C,
llist_add(C)→ head=C→B→ACPU0: claims slot 0 in B
CPU1: claims slot 0 in C
Now the head is C. Once C fills up, a new page D is allocated and becomes the new head. But B still has ~511 unused slots and will never
Quoted 8 lines
be the head again — we only ever look at
llist_first(). Those slots are permanently wasted.That's why I used struct page
*early_pfn_currentwith direct cmpxchg — the loser frees its page and retries, avoiding the waste.That said, I may be missing something. Is there a way to use llist that avoids this? I'd appreciate your thoughts.
No, I think you are right, that retry logic in llist_add() does not work for our case. I think the code you have works, but the way it uses page attributes is kinda hacky and confusing. If we have to open code the linked list handling then maybe just do this:
struct pfn_arr_hdr {
struct pfn_arr *next;
atomic_t count;
};#define PFN_ARR_SIZE ((PAGE_SIZE - sizeof(struct pfn_arr_hdr)) / sizeof(unsigned long))
struct pfn_arr {
struct pfn_arr_hdr hdr;
unsigned long pfns[PFN_ARR_SIZE];
};So, instead of allocating pages, you use kmalloc() to allocate struct pfn_arr and fill pfn_arr.pfns until it's full using pfn_arr.hdr.count to track the array size. When current pfn_arr is full, you allocate a new pfn_arr and link them via pfn_arr.hdr.next atomically, the way you are doing with pages. That seems cleaner to me. WDYT?
Quoted 205 lines
Quoted 12 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 12 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;Quoted 4 lines
nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx) break;will fix in the next version.
Quoted 56 lines
Quoted 56 lines
lib/alloc_tag.c +7
-23+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic Quoted 8 lines
lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 6 lines
lib/alloc_tag.c +3
-3-* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];This
page_address()can be done in the outer loop since the page does not change inside the inner loop.will fix in the next version.
Quoted 72 lines
Quoted 72 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +8
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags))Quoted 9 lines
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 1 line
mm/page_alloc.c +1
-0+alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);Any reason for changing the order of
alloc_tag_add_early_pfn()andalloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also callshould_record_early_pfn()from insidealloc_tag_add_early_pfn(), which would keep bothshould_record_early_pfn()and__GFP_NO_CODETAGdefinitions local toalloc_tag.c.will fix in the next version.
Thanks
Best regards.
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Hao Ge
reviewerHi Suren
Quoted 251 lines · Suren Baghdasaryan
Quoted 224 lines · Hao Ge
Hi Suren
Thanks a lot for your review and suggestions.
Quoted 64 lines · Suren Baghdasaryan
Quoted 48 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. nit: Callers must... sounds like someone who uses
__alloc_tag_add_early_pfn()has to do something extra, howeverclear_page_tag_ref()is done byclear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:Quoted 3 lines
Codetags of the pages allocated with
__GFP_NO_CODETAGshould be cleared (viaclear_page_tag_ref()) before freeing the pages to preventalloc_tag_sub_check()from triggering a warning.That's reasonable.
I will adjust the wording in v2.
Quoted 48 lines
Quoted 48 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +1
-12@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))Quoted 9 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 3 lines
lib/alloc_tag.c +1
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;So, instead of using
early_pfn_currentas the head page and thenpage->lru.nextdirectly, I would suggest having "structllist_headearly_pfn_head;" as the list head and either usingpage->pcp_llistto link the pages or even better, adding a "structllist_nodellist;" into the union containing "structlist_headlru;". Note thatllist_add()usestry_cmpxchg(), so the link addition is atomic.Adding a struct
llist_nodeinto the page union only for theCONFIG_MEM_ALLOC_PROFILING_DEBUGfeature,which may not be quite appropriate from a design perspective.
Quoted 21 lines
And you should probably use folios rather than pages to follow the new trends.
Quoted 19 lines
lib/alloc_tag.c +9
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) {Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 793
Quoted 8 lines
lib/alloc_tag.c +8
-0+pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new->lru.next = (struct list_head *)page; +if (cmpxchg (&early_pfn_current, page, new) != page) { +__free_page (new); +continue ; + }I also considered using
llist_headinitially. like thisstatic
LLIST_HEAD(early_pfn_pages);static void
__init__alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags){ struct page *page; unsigned long idx; do { page = llist_first(&early_pfn_pages); if (!page || READ_ONCE(page->private) >=
EARLY_PFNS_PER_PAGE) {
gfp_tgfp =gfp_flags& ~__GFP_DIRECT_RECLAIM; struct page *new = alloc_page(gfp |__GFP_NO_CODETAG);if (!new) {
pr_warn_once("early PFN tracking pageallocation failed\n"); return; } new->private = 0; llist_add(&new->pcp_llist, &early_pfn_pages); page = new; } idx = READ_ONCE(page->private); if (idx >= EARLY_PFNS_PER_PAGE) continue; if (cmpxchg(&page->private, idx, idx + 1) == idx) break; } while (1); ((unsigned long *)page_address(page))[idx] = pfn; }The problem is that
llist_add()retries internally until it succeeds.When multiple CPUs concurrently see that the head page is NULL or full ,
they each allocate a new tracking page and all insertions succeed:
CPU0: head page A is full or NULL, allocates B,
llist_add(B)→ head=B→ACPU1: head page A is full or NULL, allocates C,
llist_add(C)→ head=C→B→ACPU0: claims slot 0 in B
CPU1: claims slot 0 in C
Now the head is C. Once C fills up, a new page D is allocated and becomes the new head. But B still has ~511 unused slots and will never
Quoted 8 lines
be the head again — we only ever look at
llist_first(). Those slots are permanently wasted.That's why I used struct page
*early_pfn_currentwith direct cmpxchg — the loser frees its page and retries, avoiding the waste.That said, I may be missing something. Is there a way to use llist that avoids this? I'd appreciate your thoughts.
No, I think you are right, that retry logic in
llist_add()does not work for our case. I think the code you have works, but the way it uses page attributes is kinda hacky and confusing. If we have to open code the linked list handling then maybe just do this:struct pfn_arr_hdr { struct pfn_arr *next; atomic_t count; };#define
PFN_ARR_SIZE((PAGE_SIZE-sizeof(struct pfn_arr_hdr)) / sizeof(unsigned long))struct pfn_arr { struct pfn_arr_hdr hdr; unsigned long pfns[PFN_ARR_SIZE]; };
Quoted 5 lines
So, instead of allocating pages, you use
kmalloc()to allocate structpfn_arrand fill pfn_arr.pfns until it's full using pfn_arr.hdr.count to track the array size. When currentpfn_arris full, you allocate a newpfn_arrand link them via pfn_arr.hdr.next atomically, the way you are doing with pages. That seems cleaner to me. WDYT?
Actually, using kmalloc() would not be safe either, as page allocations can occur before kmalloc() becomes available.
For example:
memblock_free_all() ← buddy allocator ready
mem_init()
preallocate_vmalloc_pages()
p4d_alloc() / pud_alloc()
alloc_pages() -> triggers alloc_tag_add_early_pfn()
kmem_cache_init() -> kmalloc becomes available
Using alloc_page() is the safest choice — the very fact that we reach alloc_tag_add_early_pfn() means a page allocation via the buddy
allocator has already succeeded, so the buddy allocator is guaranteed to be ready.
How about keeping alloc_page() but mapping the page body to a proper struct?
struct pfn_pool {
struct pfn_pool *next;
atomic_t count;
unsigned long pfns[];
};
The first few bytes of the page hold metadata (next pointer and slot count), the remainder stores PFNs.
WDYT?
Thanks.
Best regards.
Hao
Quoted 204 lines
Quoted 204 lines
Quoted 12 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 12 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;Quoted 4 lines
nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx) break;will fix in the next version.
Quoted 55 lines
Quoted 55 lines
lib/alloc_tag.c +7
-23+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic Quoted 8 lines
lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 6 lines
lib/alloc_tag.c +3
-3-* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];This
page_address()can be done in the outer loop since the page does not change inside the inner loop.will fix in the next version.
Quoted 72 lines
Quoted 72 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +8
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags))Quoted 9 lines
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 1 line
mm/page_alloc.c +1
-0+alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);Any reason for changing the order of
alloc_tag_add_early_pfn()andalloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also callshould_record_early_pfn()from insidealloc_tag_add_early_pfn(), which would keep bothshould_record_early_pfn()and__GFP_NO_CODETAGdefinitions local toalloc_tag.c.will fix in the next version.
Thanks
Best regards.
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1
Suren Baghdasaryan
maintainerQuoted 256 lines · Hao Ge
Hi Suren
Quoted 251 lines · Suren Baghdasaryan
Quoted 224 lines · Hao Ge
Hi Suren
Thanks a lot for your review and suggestions.
Quoted 64 lines · Suren Baghdasaryan
Quoted 48 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pages. Each page stores PFNs in its page body and uses
page->lruto link to the next page, withpage->privatetracking the number of used slots.The list pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Signed-off-by Hao Ge <[email protected]>---
v3:
- Simplify linked list: usepage->lrufor chaining andpage->privateas slot counter, removing theearly_pfn_nodestruct and freelist (suggested by Suren Baghdasaryan)
- Passgfp_flagsthroughalloc_tag_add_early_pfn()but strip__GFP_DIRECT_RECLAIMinstead of selecting GFP_KERNEL/GFP_ATOMIC, because__alloc_tag_add_early_pfn()is invoked underrcu_read_lock().v2:
- Use cmpxchg to atomically updateearly_pfn_pages, preventing page leak under concurrent allocation
- Passgfp_flagsthrough the full call chain and usegfpflags_allow_blocking()
to selectGFP_KERNELvsGFP_ATOMIC, avoiding unnecessaryGFP_ATOMICin process context--- include/linux/alloc_tag.h | 22 ++++++- lib/alloc_tag.c | 118 +++++++++++++++++++++----------------- mm/page_alloc.c | 23 ++++---- 3 files changed, 97 insertions(+), 66 deletions(-)include/linux/alloc_tag.h +0
-0@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG include/linux/alloc_tag.h static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) · line 162
Quoted 10 lines
include/linux/alloc_tag.h +10
-0+/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. nit: Callers must... sounds like someone who uses
__alloc_tag_add_early_pfn()has to do something extra, howeverclear_page_tag_ref()is done byclear_early_alloc_pfn_tag_refs(), so callers don't really need to do this. I suggest rephrasing as:Quoted 3 lines
Codetags of the pages allocated with
__GFP_NO_CODETAGshould be cleared (viaclear_page_tag_ref()) before freeing the pages to preventalloc_tag_sub_check()from triggering a warning.That's reasonable.
I will adjust the wording in v2.
Quoted 48 lines
Quoted 48 lines
include/linux/alloc_tag.h +10
-2+*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),@@ -163,11 +180,12 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return false; }#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +1
-12@@ -766,45 +766,49 @@ static __init bool need_page_alloc_tagging(void)* Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. -* -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. -* -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define EARLY_PFNS_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))Quoted 9 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 772
Quoted 3 lines
lib/alloc_tag.c +1
-2-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct page *early_pfn_current__initdata ;So, instead of using
early_pfn_currentas the head page and thenpage->lru.nextdirectly, I would suggest having "structllist_headearly_pfn_head;" as the list head and either usingpage->pcp_llistto link the pages or even better, adding a "structllist_nodellist;" into the union containing "structlist_headlru;". Note thatllist_add()usestry_cmpxchg(), so the link addition is atomic.Adding a struct
llist_nodeinto the page union only for theCONFIG_MEM_ALLOC_PROFILING_DEBUGfeature,which may not be quite appropriate from a design perspective.
Quoted 21 lines
And you should probably use folios rather than pages to follow the new trends.
Quoted 19 lines
lib/alloc_tag.c +9
-7-static void __init __alloc_tag_add_early_pfn (unsigned long pfn) +static void __init __alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) { -int old_idx, new_idx; +struct page *page; +unsigned long idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + page = READ_ONCE(early_pfn_current); + if (!page || READ_ONCE(page->private) >= EARLY_PFNS_PER_PAGE) { + gfp_t gfp = gfp_flags & ~__GFP_DIRECT_RECLAIM; + struct page *new = alloc_page(gfp | __GFP_NO_CODETAG); + + if (!new) {Quoted 8 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 793
Quoted 8 lines
lib/alloc_tag.c +8
-0+pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new->lru.next = (struct list_head *)page; +if (cmpxchg (&early_pfn_current, page, new) != page) { +__free_page (new); +continue ; + }I also considered using
llist_headinitially. like thisstatic
LLIST_HEAD(early_pfn_pages);static void
__init__alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags){ struct page *page; unsigned long idx; do { page = llist_first(&early_pfn_pages); if (!page || READ_ONCE(page->private) >=
EARLY_PFNS_PER_PAGE) {
gfp_tgfp =gfp_flags& ~__GFP_DIRECT_RECLAIM; struct page *new = alloc_page(gfp |__GFP_NO_CODETAG);if (!new) {
pr_warn_once("early PFN tracking pageallocation failed\n"); return; } new->private = 0; llist_add(&new->pcp_llist, &early_pfn_pages); page = new; } idx = READ_ONCE(page->private); if (idx >= EARLY_PFNS_PER_PAGE) continue; if (cmpxchg(&page->private, idx, idx + 1) == idx) break; } while (1); ((unsigned long *)page_address(page))[idx] = pfn; }The problem is that
llist_add()retries internally until it succeeds.When multiple CPUs concurrently see that the head page is NULL or full ,
they each allocate a new tracking page and all insertions succeed:
CPU0: head page A is full or NULL, allocates B,
llist_add(B)→ head=B→ACPU1: head page A is full or NULL, allocates C,
llist_add(C)→ head=C→B→ACPU0: claims slot 0 in B
CPU1: claims slot 0 in C
Now the head is C. Once C fills up, a new page D is allocated and becomes the new head. But B still has ~511 unused slots and will never
Quoted 8 lines
be the head again — we only ever look at
llist_first(). Those slots are permanently wasted.That's why I used struct page
*early_pfn_currentwith direct cmpxchg — the loser frees its page and retries, avoiding the waste.That said, I may be missing something. Is there a way to use llist that avoids this? I'd appreciate your thoughts.
No, I think you are right, that retry logic in
llist_add()does not work for our case. I think the code you have works, but the way it uses page attributes is kinda hacky and confusing. If we have to open code the linked list handling then maybe just do this:struct pfn_arr_hdr { struct pfn_arr *next; atomic_t count; };#define
PFN_ARR_SIZE((PAGE_SIZE-sizeof(struct pfn_arr_hdr)) / sizeof(unsigned long))struct pfn_arr { struct pfn_arr_hdr hdr; unsigned long pfns[PFN_ARR_SIZE]; };
Quoted 8 lines
Quoted 5 lines
So, instead of allocating pages, you use
kmalloc()to allocate structpfn_arrand fill pfn_arr.pfns until it's full using pfn_arr.hdr.count to track the array size. When currentpfn_arris full, you allocate a newpfn_arrand link them via pfn_arr.hdr.next atomically, the way you are doing with pages. That seems cleaner to me. WDYT?Actually, using
kmalloc()would not be safe either, as page allocations can occur beforekmalloc()becomes available.
I see.
Quoted 31 lines
For example:
memblock_free_all()← buddy allocator ready
mem_init()preallocate_vmalloc_pages() p4d_alloc() / pud_alloc() alloc_pages() -> triggers alloc_tag_add_early_pfn()
kmem_cache_init()-> kmalloc becomes availableUsing
alloc_page()is the safest choice — the very fact that we reachalloc_tag_add_early_pfn()means a page allocation via the buddyallocator has already succeeded, so the buddy allocator is guaranteed to be ready.
How about keeping
alloc_page()but mapping the page body to a proper struct?struct
pfn_pool{struct pfn_pool *next; atomic_t count; unsigned long pfns[];
Quoted 6 lines
};
The first few bytes of the page hold metadata (next pointer and slot count), the remainder stores PFNs.
WDYT?
Yeah, that sounds good to me.
Quoted 211 lines
Thanks.
Best regards.
Hao
Quoted 204 lines
Quoted 204 lines
Quoted 12 lines
lib/alloc_tag.c static __init bool need_page_alloc_tagging(void) · line 801
Quoted 12 lines
lib/alloc_tag.c +7
-2+ page = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =READ_ONCE (page->private); +if (idx >=EARLY_PFNS_PER_PAGE ) +continue ; +if (cmpxchg (&page->private, idx, idx +1 ) != idx) +continue ; +break ;Quoted 4 lines
nit: This would read more easily:
if (cmpxchg(&page->private, idx, idx + 1) == idx) break;will fix in the next version.
Quoted 55 lines
Quoted 55 lines
lib/alloc_tag.c +7
-23+ }while (1 ); - early_pfns[old_idx] = pfn; + ((unsigned long *)page_address (page))[idx] = pfn; } -typedef void alloc_tag_add_func (unsigned long pfn); +typedef void alloc_tag_add_func (unsigned long pfn,gfp_t gfp_flags);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;@@ -814,13 +818,14 @@ void alloc_tag_add_early_pfn(unsigned long pfn)rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add) -alloc_tag_add (pfn); +alloc_tag_add (pfn, gfp_flags);rcu_read_unlock (); }static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct page *page, *next; +unsigned long i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +834,42 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic Quoted 8 lines
lib/alloc_tag.c static void __init clear_early_alloc_pfn_tag_refs(void) · line 839
Quoted 6 lines
lib/alloc_tag.c +3
-3-* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (page = early_pfn_current; page; page = next) { +for (i =0 ; i < page->private; i++) { +unsigned long pfn = ((unsigned long *)page_address (page))[i];This
page_address()can be done in the outer loop since the page does not change inside the inner loop.will fix in the next version.
Quoted 72 lines
Quoted 72 lines
lib/alloc_tag.c +28
-5+ +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); - continue; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = (struct page *)page->lru.next; +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +8
-6@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1294,21 +1294,24 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { -/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. -*/ -alloc_tag_add_early_pfn (page_to_pfn (page)); +if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, record the pfn so the +* tag ref can be cleared later when page_ext is initialized. +*/ +if (should_record_early_pfn (gfp_flags))Quoted 9 lines
mm/page_alloc.c void __pgalloc_tag_add(struct page *page, struct task_struct *task, · line 1306
Quoted 1 line
mm/page_alloc.c +1
-0+alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);Any reason for changing the order of
alloc_tag_add_early_pfn()andalloc_tag_set_inaccurate()? I liked the previous version where we explain why this is done at the very beginning of the block. You could also callshould_record_early_pfn()from insidealloc_tag_add_early_pfn(), which would keep bothshould_record_early_pfn()and__GFP_NO_CODETAGdefinitions local toalloc_tag.c.will fix in the next version.
Thanks
Best regards.
Hao
Quoted 34 lines
Quoted 34 lines
mm/page_alloc.c +4
-5} }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1341,7 +1344,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1896,7 +1899,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, --2.25.1