mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Subsystems (4)
Hao Ge
reviewerPATCH 1/1 mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Pages allocated before page_ext is available have their codetag left uninitialized. Track these early PFNs and clear their codetag in clear_early_alloc_pfn_tag_refs() to avoid "alloc_tag was not set" warnings when they are freed later.
Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list. Each page is carved into early_pfn_node entries and the remainder is kept as a freelist for subsequent allocations.
The list nodes themselves are allocated via alloc_page(), which would trigger __pgalloc_tag_add() -> alloc_tag_add_early_pfn() -> alloc_early_pfn_node() and recurse indefinitely. Introduce __GFP_NO_CODETAG (aliased to __GFP_NO_OBJ_EXT) and pass gfp_flags through pgalloc_tag_add() so that the early path can skip recording allocations that carry this flag.
--- include/linux/alloc_tag.h | 18 ++++++++ lib/alloc_tag.c | 91 ++++++++++++++++++++++++++------------- mm/page_alloc.c | 27 ++++++++---- 3 files changed, 97 insertions(+), 39 deletions(-)
include/linux/alloc_tag.h +18 -0
@@ -150,6 +150,23 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) }#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by alloc_early_pfn_node() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. +* +* Callers must set the codetag to CODETAG_EMPTY (via +* clear_page_tag_ref()) before freeing pages allocated with this +* flag once page_ext becomes available, otherwise +* alloc_tag_sub_check() will trigger a warning. +*/ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static inline bool should_record_early_pfn (gfp_t gfp_flags) +{ +return !(gfp_flags & __GFP_NO_CODETAG); +}static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) { WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),
@@ -168,6 +185,7 @@ void alloc_tag_add_early_pfn(unsigned long pfn);static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {}static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline bool should_record_early_pfn (gfp_t gfp_flags) {return true; }#endif /* Caller should verify both ref and tag to be valid */
lib/alloc_tag.c +61 -30
126 lines not on this page · raw message
mm/page_alloc.c +18 -9
@@ -1284,7 +1284,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;
@@ -1294,21 +1294,30 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); }else { + +if (task->alloc_tag) +alloc_tag_set_inaccurate (task->alloc_tag); + +/* +* page_ext is not available yet, skip if this allocation +* doesn't need early PFN recording. +*/ +if (unlikely (!should_record_early_pfn (gfp_flags))) +return ; +/* -* page_ext is not available yet, record the pfn so we can -* clear the tag ref later when page_ext is initialized. +* Record the pfn so the tag ref can be cleared later +* when page_ext is initialized. */ alloc_tag_add_early_pfn (page_to_pfn (page)); -if (task->alloc_tag) -alloc_tag_set_inaccurate (task->alloc_tag); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */
@@ -1341,7 +1350,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}
@@ -1896,7 +1905,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Signature · 1 line
2.25.1