mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Subsystems (4)
Hao Ge
reviewerPATCH v6 1/1 mm/alloc_tag: replace fixed-size early PFN array with dynamic linked list
Pages allocated before page_ext is available have their codetag left uninitialized. Track these early PFNs and clear their codetag in clear_early_alloc_pfn_tag_refs() to avoid "alloc_tag was not set" warnings when they are freed later.
Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of pfn_pool structs. Each node is allocated via alloc_page() and mapped to a pfn_pool containing a next pointer, an atomic slot counter, and a PFN array that fills the remainder of the page.
The tracking pages themselves are allocated via alloc_page(), which would trigger __pgalloc_tag_add() -> alloc_tag_add_early_pfn() and recurse indefinitely. Introduce __GFP_NO_CODETAG (reuses the %__GFP_NO_OBJ_EXT bit) and pass gfp_flags through pgalloc_tag_add() so that the early path can skip recording allocations that carry this flag.
--- include/linux/alloc_tag.h | 4 +- lib/alloc_tag.c | 140 +++++++++++++++++++++++++------------- mm/page_alloc.c | 12 ++-- 3 files changed, 99 insertions(+), 57 deletions(-)
include/linux/alloc_tag.h +2 -2
@@ -163,11 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {}#endif /* Caller should verify both ref and tag to be valid */
lib/alloc_tag.c +91 -49
185 lines not on this page · raw message
mm/page_alloc.c +6 -6
@@ -1240,7 +1240,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;
@@ -1254,17 +1254,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,* page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ -alloc_tag_add_early_pfn (page_to_pfn (page)); +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */
@@ -1297,7 +1297,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}
@@ -1852,7 +1852,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Signature · 1 line
2.25.1
Hao Ge
reviewerAdd cc:[email protected] and lost part of the changelog
because of local Git environment glitch.
Sorry for this.
Quoted 14 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of
pfn_poolstructs. Each node is allocated viaalloc_page()and mapped to apfn_poolcontaining a next pointer, an atomic slot counter, and a PFN array that fills the remainder of the page.
Quoted 8 lines
The tracking pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Suggested-by Suren Baghdasaryan <[email protected]>Signed-off-by Hao Ge <[email protected]>
---
v6:
- Use hardcoded __GFP_HIGH | __GFP_NO_CODETAG instead of inheriting caller's gfp_flags for internal pfn_pool page allocation.
Quoted 268 lines
--- include/linux/alloc_tag.h | 4 +- lib/alloc_tag.c | 140 +++++++++++++++++++++++++------------- mm/page_alloc.c | 12 ++-- 3 files changed, 99 insertions(+), 57 deletions(-)include/linux/alloc_tag.h +2
-2@@ -163,11 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {}#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +91
-49@@ -767,50 +767,82 @@ static __init bool need_page_alloc_tagging(void)* their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. * -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. +* Each page is cast to a pfn_pool: the first few bytes hold metadata +* (next pointer and slot count), the remainder stores PFNs. +*/ +struct pfn_pool { +struct pfn_pool *next; +atomic_t count; +unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. * -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. +* Codetags of the pages allocated with __GFP_NO_CODETAG should be +* cleared (via clear_page_tag_ref()) before freeing the pages to prevent +* alloc_tag_sub_check() from triggering a warning. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct pfn_pool *current_pfn_pool__initdata ;static void __init __alloc_tag_add_early_pfn (unsigned long pfn) { -int old_idx, new_idx; +struct pfn_pool *pool; +int idx;do { - old_idx =atomic_read (&early_pfn_count); -if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; + pool =READ_ONCE (current_pfn_pool); +if (!pool ||atomic_read (&pool->count) >=PFN_POOL_SIZE ) { +struct page *new_page =alloc_page (__GFP_HIGH | __GFP_NO_CODETAG); +struct pfn_pool *new; + +if (!new_page) { +pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new =page_address (new_page); + new->next = pool; +atomic_set (&new->count,0 ); +if (cmpxchg (¤t_pfn_pool, pool, new) != pool) { +clear_page_tag_ref (new_page); +__free_page (new_page); +continue ; + } + pool = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =atomic_read (&pool->count); +if (idx >=PFN_POOL_SIZE ) +continue ; +if (atomic_cmpxchg (&pool->count, idx, idx +1 ) == idx) +break ; + }while (1 ); - early_pfns[old_idx] = pfn; + pool->pfns[idx] = pfn; }typedef void alloc_tag_add_func (unsigned long pfn);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;if (static_key_enabled (&mem_profiling_compressed))return ; +/* Skip allocations for the tracking list itself to avoid recursion. */ +if (gfp_flags & __GFP_NO_CODETAG) +return ; +rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add)@@ -820,7 +852,9 @@ void alloc_tag_add_early_pfn(unsigned long pfn)static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct pfn_pool *pool, *next; +struct page *page; +int i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +863,45 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic -* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (pool = current_pfn_pool; pool; pool = next) { +int nr_pfns =atomic_read (&pool->count); + +for (i =0 ; i < nr_pfns; i++) { +unsigned long pfn = pool->pfns[i]; + +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); -continue ; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = pool->next; + page =virt_to_page (pool); +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +6
-6@@ -1240,7 +1240,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1254,17 +1254,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,* page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ -alloc_tag_add_early_pfn (page_to_pfn (page)); +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1297,7 +1297,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1852,7 +1852,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Quoted 32 lines · Hao Ge
Add cc:[email protected] and lost part of the changelog
because of local Git environment glitch.
Sorry for this.
Quoted 22 lines · Hao Ge
Pages allocated before
page_extis available have their codetag left uninitialized. Track these early PFNs and clear their codetag inclear_early_alloc_pfn_tag_refs()to avoid "alloc_tagwas not set" warnings when they are freed later.Currently a fixed-size array of 8192 entries is used, with a warning if the limit is exceeded. However, the number of early allocations depends on the number of CPUs and can be larger than 8192.
Replace the fixed-size array with a dynamically allocated linked list of
pfn_poolstructs. Each node is allocated viaalloc_page()and mapped to apfn_poolcontaining a next pointer, an atomic slot counter, and a PFN array that fills the remainder of the page.The tracking pages themselves are allocated via
alloc_page(), which would trigger__pgalloc_tag_add()->alloc_tag_add_early_pfn()and recurse indefinitely. Introduce__GFP_NO_CODETAG(reuses the %__GFP_NO_OBJ_EXT bit) and passgfp_flagsthroughpgalloc_tag_add()so that the early path can skip recording allocations that carry this flag.Suggested-by Suren Baghdasaryan <[email protected]>Signed-off-by Hao Ge <[email protected]>
Quoted 275 lines
---
v6:
- Use hardcoded__GFP_HIGH|__GFP_NO_CODETAGinstead of inheriting caller'sgfp_flagsfor internalpfn_poolpage allocation.Quoted 268 lines
--- include/linux/alloc_tag.h | 4 +- lib/alloc_tag.c | 140 +++++++++++++++++++++++++------------- mm/page_alloc.c | 12 ++-- 3 files changed, 99 insertions(+), 57 deletions(-)include/linux/alloc_tag.h +2
-2@@ -163,11 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {WARN_ONCE (ref && !ref->ct,"alloc_tag was not set\n" ); } -void alloc_tag_add_early_pfn (unsigned long pfn); +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags);#else static inline void alloc_tag_add_check (union codetag_ref *ref,struct alloc_tag *tag) {}static inline void alloc_tag_sub_check (union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn (unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {}#endif /* Caller should verify both ref and tag to be valid */ lib/alloc_tag.c +91
-49@@ -767,50 +767,82 @@ static __init bool need_page_alloc_tagging(void)* their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. * -* Early allocations include: -* - Base allocations independent of CPU count -* - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, -* such as trace ring buffers, scheduler per-cpu data) -* -* For simplicity, we fix the size to 8192. -* If insufficient, a warning will be triggered to alert the user. +* Each page is cast to a pfn_pool: the first few bytes hold metadata +* (next pointer and slot count), the remainder stores PFNs. +*/ +struct pfn_pool { +struct pfn_pool *next; +atomic_t count; +unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* +* Skip early PFN recording for a page allocation. Reuses the +* %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid +* recursion when allocating pages for the early PFN tracking list +* itself. * -* TODO: Replace fixed-size array with dynamic allocation using -* a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. +* Codetags of the pages allocated with __GFP_NO_CODETAG should be +* cleared (via clear_page_tag_ref()) before freeing the pages to prevent +* alloc_tag_sub_check() from triggering a warning. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX ]__initdata ; -static atomic_t early_pfn_count__initdata =ATOMIC_INIT (0 ); +static struct pfn_pool *current_pfn_pool__initdata ;static void __init __alloc_tag_add_early_pfn (unsigned long pfn) { -int old_idx, new_idx; +struct pfn_pool *pool; +int idx;do { - old_idx =atomic_read (&early_pfn_count); -if (old_idx >=EARLY_ALLOC_PFN_MAX ) { -pr_warn_once ("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n" , -EARLY_ALLOC_PFN_MAX ); -return ; + pool =READ_ONCE (current_pfn_pool); +if (!pool ||atomic_read (&pool->count) >=PFN_POOL_SIZE ) { +struct page *new_page =alloc_page (__GFP_HIGH | __GFP_NO_CODETAG); +struct pfn_pool *new; + +if (!new_page) { +pr_warn_once ("early PFN tracking page allocation failed\n" ); +return ; + } + new =page_address (new_page); + new->next = pool; +atomic_set (&new->count,0 ); +if (cmpxchg (¤t_pfn_pool, pool, new) != pool) { +clear_page_tag_ref (new_page); +__free_page (new_page); +continue ; + } + pool = new; } - new_idx = old_idx +1 ; - }while (!atomic_try_cmpxchg (&early_pfn_count, &old_idx, new_idx)); + idx =atomic_read (&pool->count); +if (idx >=PFN_POOL_SIZE ) +continue ; +if (atomic_cmpxchg (&pool->count, idx, idx +1 ) == idx) +break ; + }while (1 ); - early_pfns[old_idx] = pfn; + pool->pfns[idx] = pfn; }typedef void alloc_tag_add_func (unsigned long pfn);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr__refdata =RCU_INITIALIZER (__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn (unsigned long pfn) +void alloc_tag_add_early_pfn (unsigned long pfn,gfp_t gfp_flags) {alloc_tag_add_func *alloc_tag_add;if (static_key_enabled (&mem_profiling_compressed))return ; +/* Skip allocations for the tracking list itself to avoid recursion. */ +if (gfp_flags & __GFP_NO_CODETAG) +return ; +rcu_read_lock (); alloc_tag_add =rcu_dereference (alloc_tag_add_early_pfn_ptr);if (alloc_tag_add)@@ -820,7 +852,9 @@ void alloc_tag_add_early_pfn(unsigned long pfn)static void __init clear_early_alloc_pfn_tag_refs (void ) { -unsigned int i; +struct pfn_pool *pool, *next; +struct page *page; +int i;if (static_key_enabled (&mem_profiling_compressed))return ;@@ -829,37 +863,45 @@ static void __init clear_early_alloc_pfn_tag_refs(void)/* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu (); -for (i =0 ; i <atomic_read (&early_pfn_count); i++) { -unsigned long pfn = early_pfns[i]; - -if (pfn_valid (pfn)) { -struct page *page =pfn_to_page (pfn); -union pgtag_ref_handle handle; -union codetag_ref ref; - -if (get_page_tag_ref (page, &ref, &handle)) { -/* -* An early-allocated page could be freed and reallocated -* after its page_ext is initialized but before we clear it. -* In that case, it already has a valid tag set. -* We should not overwrite that valid tag with CODETAG_EMPTY. -* -* Note: there is still a small race window between checking -* ref.ct and calling set_codetag_empty(). We accept this -* race as it's unlikely and the extra complexity of atomic -* cmpxchg is not worth it for this debug-only code path. -*/ -if (ref.ct) { +for (pool = current_pfn_pool; pool; pool = next) { +int nr_pfns =atomic_read (&pool->count); + +for (i =0 ; i < nr_pfns; i++) { +unsigned long pfn = pool->pfns[i]; + +if (pfn_valid (pfn)) { +union pgtag_ref_handle handle; +union codetag_ref ref; + +if (get_page_tag_ref (pfn_to_page (pfn), &ref, &handle)) { +/* +* An early-allocated page could be freed and reallocated +* after its page_ext is initialized but before we clear it. +* In that case, it already has a valid tag set. +* We should not overwrite that valid tag +* with CODETAG_EMPTY. +* +* Note: there is still a small race window between checking +* ref.ct and calling set_codetag_empty(). We accept this +* race as it's unlikely and the extra complexity of atomic +* cmpxchg is not worth it for this debug-only code path. +*/ +if (ref.ct) { +put_page_tag_ref (handle); +continue ; + } + +set_codetag_empty (&ref); +update_page_tag_ref (handle, &ref);put_page_tag_ref (handle); -continue ; } - -set_codetag_empty (&ref); -update_page_tag_ref (handle, &ref); -put_page_tag_ref (handle); } } + next = pool->next; + page =virt_to_page (pool); +clear_page_tag_ref (page); +__free_page (page); } }#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ mm/page_alloc.c +6
-6@@ -1240,7 +1240,7 @@ void __clear_page_tag_ref(struct page *page)/* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {union pgtag_ref_handle handle;union codetag_ref ref;@@ -1254,17 +1254,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,* page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ -alloc_tag_add_early_pfn (page_to_pfn (page)); +alloc_tag_add_early_pfn (page_to_pfn (page), gfp_flags);if (task->alloc_tag)alloc_tag_set_inaccurate (task->alloc_tag); } }static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) +unsigned int nr,gfp_t gfp_flags) {if (mem_alloc_profiling_enabled ()) -__pgalloc_tag_add (page, task, nr); +__pgalloc_tag_add (page, task, nr, gfp_flags); }/* Should be called only if mem_alloc_profiling_enabled() */ @@ -1297,7 +1297,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)#else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add (struct page *page,struct task_struct *task, -unsigned int nr) {} +unsigned int nr,gfp_t gfp_flags) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {}static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}@@ -1852,7 +1852,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,set_page_owner (page, order, gfp_flags);page_table_check_alloc (page, order); -pgalloc_tag_add (page, current,1 << order); +pgalloc_tag_add (page, current,1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,