mm/codetag: fix a typo
Subsystems (7)
PATCH 1/3 mm/codetag: fix a typo
--- include/linux/alloc_tag.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/alloc_tag.h +1 -1
@@ -70,7 +70,7 @@ static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)/* * When percpu variables are required to be defined as weak, static percpu * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION). -* Instead we will accound all module allocations to a single counter. +* Instead we will account all module allocations to a single counter. */ DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
PATCH 2/3 mm/codetag: fix pgalloc_tag_split()
Only tag the new head pages when splitting one large folio to multiple ones of a lower order. Tagging tail pages can cause imbalanced "calls" counters, since only head pages are untagged by pgalloc_tag_sub() and reference counts on tail pages are leaked, e.g.,
# echo 2048kB >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote_size # echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages # time echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote # grep alloc_gigantic_folio /proc/allocinfo
Before this patch:
0 549427200 mm/hugetlb.c:1549 func:alloc_gigantic_folio
real 0m2.057s user 0m0.000s sys 0m2.051s
After this patch:
0 0 mm/hugetlb.c:1549 func:alloc_gigantic_folio
real 0m1.711s user 0m0.000s sys 0m1.704s
Not tagging tail pages also improves the splitting time, e.g., by about 15% when demoting 1GB hugeTLB folios to 2MB ones, as shown above.
--- include/linux/mm.h | 30 ++++++++++++++++++++++++++++++ include/linux/pgalloc_tag.h | 31 ------------------------------- mm/huge_memory.c | 2 +- mm/hugetlb.c | 2 +- mm/page_alloc.c | 4 ++-- 5 files changed, 34 insertions(+), 35 deletions(-)
include/linux/mm.h +30 -0
@@ -4137,4 +4137,34 @@ void vma_pgtable_walk_end(struct vm_area_struct *vma);int reserve_mem_find_by_name (const char *name,phys_addr_t *start,phys_addr_t *size); +#ifdef CONFIG_MEM_ALLOC_PROFILING +static inline void pgalloc_tag_split (struct folio *folio,int old_order,int new_order) +{ +int i; +struct alloc_tag *tag; +unsigned int nr_pages =1 << new_order; + +if (!mem_alloc_profiling_enabled ()) +return ; + + tag =pgalloc_tag_get (&folio->page); +if (!tag) +return ; + +for (i = nr_pages; i < (1 << old_order); i += nr_pages) { +union codetag_ref *ref =get_page_tag_ref (folio_page (folio, i)); + +if (ref) { +/* Set new reference to point to the original tag */ +alloc_tag_ref_set (ref, tag); +put_page_tag_ref (ref); + } + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING */ +static inline void pgalloc_tag_split (struct folio *folio,int old_order,int new_order) +{ +} +#endif /* CONFIG_MEM_ALLOC_PROFILING */ +#endif /* _LINUX_MM_H */
include/linux/pgalloc_tag.h +0 -31
@@ -80,36 +80,6 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) } } -static inline void pgalloc_tag_split (struct page *page,unsigned int nr) -{ -int i; -struct page_ext *first_page_ext; -struct page_ext *page_ext; -union codetag_ref *ref; -struct alloc_tag *tag; - -if (!mem_alloc_profiling_enabled ()) -return ; - - first_page_ext = page_ext =page_ext_get (page); -if (unlikely (!page_ext)) -return ; - - ref =codetag_ref_from_page_ext (page_ext); -if (!ref->ct) - goto out; - - tag =ct_to_alloc_tag (ref->ct); - page_ext =page_ext_next (page_ext); -for (i =1 ; i < nr; i++) { -/* Set new reference to point to the original tag */ -alloc_tag_ref_set (codetag_ref_from_page_ext (page_ext), tag); - page_ext =page_ext_next (page_ext); - } -out: -page_ext_put (first_page_ext); -} -static inline struct alloc_tag *pgalloc_tag_get (struct page *page) {struct alloc_tag *tag =NULL ;
@@ -142,7 +112,6 @@ static inline void clear_page_tag_ref(struct page *page) {}static inline void pgalloc_tag_add (struct page *page,struct task_struct *task,unsigned int nr) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {} -static inline void pgalloc_tag_split (struct page *page,unsigned int nr) {}static inline struct alloc_tag *pgalloc_tag_get (struct page *page) {return NULL ; }static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}
mm/huge_memory.c +1 -1
@@ -3244,7 +3244,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,/* Caller disabled irqs, so they are still disabled here */ split_page_owner (head, order, new_order); -pgalloc_tag_split (head,1 << order); +pgalloc_tag_split (folio, order, new_order);/* See comment in __split_huge_page_tail() */ if (folio_test_anon (folio)) {
mm/hugetlb.c +1 -1
@@ -3778,7 +3778,7 @@ static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,list_del (&folio->lru);split_page_owner (&folio->page,huge_page_order (src),huge_page_order (dst)); -pgalloc_tag_split (&folio->page,1 <<huge_page_order (src)); +pgalloc_tag_split (folio,huge_page_order (src),huge_page_order (dst));for (i =0 ; i <pages_per_huge_page (src); i +=pages_per_huge_page (dst)) {struct page *page =folio_page (folio, i);
mm/page_alloc.c +2 -2
@@ -2822,7 +2822,7 @@ void split_page(struct page *page, unsigned int order)for (i =1 ; i < (1 << order); i++)set_page_refcounted (page + i);split_page_owner (page, order,0 ); -pgalloc_tag_split (page,1 << order); +pgalloc_tag_split (page_folio (page), order,0 );split_page_memcg (page, order,0 ); }EXPORT_SYMBOL_GPL (split_page);
@@ -5020,7 +5020,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order,struct page *last = page + nr;split_page_owner (page, order,0 ); -pgalloc_tag_split (page,1 << order); +pgalloc_tag_split (page_folio (page), order,0 );split_page_memcg (page, order,0 );while (page < --last)set_page_refcounted (last);
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
Re: [PATCH mm-unstable v1 2/3] mm/codetag: fix pgalloc_tag_split()
Quoted 131 lines · Yu Zhao
Only tag the new head pages when splitting one large folio to multiple ones of a lower order. Tagging tail pages can cause imbalanced "calls" counters, since only head pages are untagged by
pgalloc_tag_sub()and reference counts on tail pages are leaked, e.g.,# echo 2048kB >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote_size # echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages # time echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote # grep alloc_gigantic_folio /proc/allocinfoBefore this patch:
0 549427200mm/hugetlb.c:1549func:alloc_gigantic_folioreal 0m2.057s user 0m0.000s sys 0m2.051sAfter this patch:
0 0mm/hugetlb.c:1549func:alloc_gigantic_folioreal 0m1.711s user 0m0.000s sys 0m1.704sNot tagging tail pages also improves the splitting time, e.g., by about 15% when demoting 1GB hugeTLB folios to 2MB ones, as shown above.
Fixes be25d1d4e822 ("mm: create new codetag references during page splitting")Signed-off-by Yu Zhao <[email protected]>--- include/linux/mm.h | 30 ++++++++++++++++++++++++++++++ include/linux/pgalloc_tag.h | 31 ------------------------------- mm/huge_memory.c | 2 +- mm/hugetlb.c | 2 +- mm/page_alloc.c | 4 ++-- 5 files changed, 34 insertions(+), 35 deletions(-)include/linux/mm.h +30
-0@@ -4137,4 +4137,34 @@ void vma_pgtable_walk_end(struct vm_area_struct *vma);int reserve_mem_find_by_name (const char *name,phys_addr_t *start,phys_addr_t *size); +#ifdef CONFIG_MEM_ALLOC_PROFILING +static inline void pgalloc_tag_split (struct folio *folio,int old_order,int new_order) +{ +int i; +struct alloc_tag *tag; +unsigned int nr_pages =1 << new_order; + +if (!mem_alloc_profiling_enabled ()) +return ; + + tag =pgalloc_tag_get (&folio->page); +if (!tag) +return ; + +for (i = nr_pages; i < (1 << old_order); i += nr_pages) { +union codetag_ref *ref =get_page_tag_ref (folio_page (folio, i)); + +if (ref) { +/* Set new reference to point to the original tag */ +alloc_tag_ref_set (ref, tag); +put_page_tag_ref (ref); + } + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING */ +static inline void pgalloc_tag_split (struct folio *folio,int old_order,int new_order) +{ +} +#endif /* CONFIG_MEM_ALLOC_PROFILING */ +#endif /* _LINUX_MM_H */ include/linux/pgalloc_tag.h +0
-31@@ -80,36 +80,6 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) } } -static inline void pgalloc_tag_split (struct page *page,unsigned int nr) -{ -int i; -struct page_ext *first_page_ext; -struct page_ext *page_ext; -union codetag_ref *ref; -struct alloc_tag *tag; - -if (!mem_alloc_profiling_enabled ()) -return ; - - first_page_ext = page_ext =page_ext_get (page); -if (unlikely (!page_ext)) -return ; - - ref =codetag_ref_from_page_ext (page_ext); -if (!ref->ct) - goto out; - - tag =ct_to_alloc_tag (ref->ct); - page_ext =page_ext_next (page_ext); -for (i =1 ; i < nr; i++) { -/* Set new reference to point to the original tag */ -alloc_tag_ref_set (codetag_ref_from_page_ext (page_ext), tag); - page_ext =page_ext_next (page_ext); - } -out: -page_ext_put (first_page_ext); -} -static inline struct alloc_tag *pgalloc_tag_get (struct page *page) {struct alloc_tag *tag =NULL ;@@ -142,7 +112,6 @@ static inline void clear_page_tag_ref(struct page *page) {}static inline void pgalloc_tag_add (struct page *page,struct task_struct *task,unsigned int nr) {}static inline void pgalloc_tag_sub (struct page *page,unsigned int nr) {} -static inline void pgalloc_tag_split (struct page *page,unsigned int nr) {}static inline struct alloc_tag *pgalloc_tag_get (struct page *page) {return NULL ; }static inline void pgalloc_tag_sub_pages (struct alloc_tag *tag,unsigned int nr) {}mm/huge_memory.c +0
-0
mm/huge_memory.c static void __split_huge_page(struct page *page, struct list_head *list, · line 3247
Quoted 5 lines
mm/huge_memory.c +1
-1@@ -3244,7 +3244,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,/* Caller disabled irqs, so they are still disabled here */ split_page_owner (head, order, new_order); -pgalloc_tag_split (head,1 << order); +pgalloc_tag_split (folio, order, new_order);
Looks like here and in the next diff you are fixing an additional bug. I was assuming that we are always splitting into order-0 pages, which is not the case anymore. It's worth mentioning that in the changelog. With that added:
Quoted 41 lines
mm/huge_memory.c +0
-0/* See comment in __split_huge_page_tail() */ if (folio_test_anon (folio)) {mm/hugetlb.c +1
-1@@ -3778,7 +3778,7 @@ static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,list_del (&folio->lru);split_page_owner (&folio->page,huge_page_order (src),huge_page_order (dst)); -pgalloc_tag_split (&folio->page,1 <<huge_page_order (src)); +pgalloc_tag_split (folio,huge_page_order (src),huge_page_order (dst));for (i =0 ; i <pages_per_huge_page (src); i +=pages_per_huge_page (dst)) {struct page *page =folio_page (folio, i);mm/page_alloc.c +2
-3@@ -2822,7 +2822,7 @@ void split_page(struct page *page, unsigned int order)for (i =1 ; i < (1 << order); i++)set_page_refcounted (page + i);split_page_owner (page, order,0 ); -pgalloc_tag_split (page,1 << order); +pgalloc_tag_split (page_folio (page), order,0 );split_page_memcg (page, order,0 ); }EXPORT_SYMBOL_GPL (split_page);@@ -5020,7 +5020,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order,struct page *last = page + nr;split_page_owner (page, order,0 ); -pgalloc_tag_split (page,1 << order); +pgalloc_tag_split (page_folio (page), order,0 );split_page_memcg (page, order,0 );while (page < --last)set_page_refcounted (last); --2.46.0.469.g59c65b2a67-goog
PATCH 3/3 mm/codetag: add pgalloc_tag_copy()
In reply to Yu ZhaoAdd pgalloc_tag_copy() to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump into compaction_alloc, e.g.,
# echo 1 >/proc/sys/vm/compact_memory # grep compaction_alloc /proc/allocinfo
Before this patch:
132968448 32463 mm/compaction.c:1880 func:compaction_alloc
After this patch:
0 0 mm/compaction.c:1880 func:compaction_alloc
--- include/linux/alloc_tag.h | 24 ++++++++++-------------- include/linux/mm.h | 25 +++++++++++++++++++++++++ mm/migrate.c | 1 + 3 files changed, 36 insertions(+), 14 deletions(-)
include/linux/alloc_tag.h +10 -14
@@ -137,7 +137,16 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {}/* Caller should verify both ref and tag to be valid */ static inline void __alloc_tag_ref_set (union codetag_ref *ref,struct alloc_tag *tag) { +alloc_tag_add_check (ref, tag); +if (!ref || !tag) +return ; + ref->ct = &tag->ct; +} + +static inline void alloc_tag_ref_set (union codetag_ref *ref,struct alloc_tag *tag) +{ +__alloc_tag_ref_set (ref, tag);/* * We need in increment the call counter every time we have a new * allocation or when we split a large allocation into smaller ones.
@@ -147,22 +156,9 @@ static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tagthis_cpu_inc (tag->counters->calls); } -static inline void alloc_tag_ref_set (union codetag_ref *ref,struct alloc_tag *tag) -{ -alloc_tag_add_check (ref, tag); -if (!ref || !tag) -return ; - -__alloc_tag_ref_set (ref, tag); -} -static inline void alloc_tag_add (union codetag_ref *ref,struct alloc_tag *tag,size_t bytes) { -alloc_tag_add_check (ref, tag); -if (!ref || !tag) -return ; - -__alloc_tag_ref_set (ref, tag); +alloc_tag_ref_set (ref, tag);this_cpu_add (tag->counters->bytes, bytes); }
include/linux/mm.h +25 -0
@@ -4161,10 +4161,35 @@ static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new } } } + +static inline void pgalloc_tag_copy (struct folio *new,struct folio *old) +{ +struct alloc_tag *tag; +union codetag_ref *ref; + + tag =pgalloc_tag_get (&old->page); +if (!tag) +return ; + + ref =get_page_tag_ref (&new->page); +if (!ref) +return ; + +/* Clear the old ref to the original allocation site. */ +clear_page_tag_ref (&old->page); +/* Decrement the counters of the tag on get_new_folio. */ +alloc_tag_sub (ref,folio_nr_pages (new)); +__alloc_tag_ref_set (ref, tag); +put_page_tag_ref (ref); +}#else /* !CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_split (struct folio *folio,int old_order,int new_order) { } + +static inline void pgalloc_tag_copy (struct folio *new,struct folio *old) +{ +}#endif /* CONFIG_MEM_ALLOC_PROFILING */ #endif /* _LINUX_MM_H */
mm/migrate.c +1 -0
@@ -750,6 +750,7 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)folio_set_readahead (newfolio);folio_copy_owner (newfolio, folio); +pgalloc_tag_copy (newfolio, folio);mem_cgroup_migrate (folio, newfolio); }
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
Andrew Morton
maintainerRe: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()
Quoted 12 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:
Add
pgalloc_tag_copy()to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump intocompaction_alloc, e.g.,# echo 1 >/proc/sys/vm/compact_memory # grep compaction_alloc /proc/allocinfoBefore this patch:
132968448 32463mm/compaction.c:1880func:compaction_allocAfter this patch:
0 0mm/compaction.c:1880func:compaction_alloc
I'm thinking that [2/3] should be backported?
And possibly this one, but for that we should identify a Fixes:, please.
Suren Baghdasaryan
maintainerRe: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()
Quoted 9 lines · Andrew Morton
Quoted 6 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:
Add
pgalloc_tag_copy()to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump intocompaction_alloc, e.g.,# echo 1 >/proc/sys/vm/compact_memory # grep compaction_alloc /proc/allocinfo
Quoted 8 lines
Quoted 6 lines
Before this patch:
132968448 32463mm/compaction.c:1880func:compaction_allocAfter this patch:
0 0mm/compaction.c:1880func:compaction_allocI'm thinking that [2/3] should be backported?
Yes, should be CC'ed to stable #6.10
Quoted 2 lines
And possibly this one, but for that we should identify a Fixes:, please.
I think for this one Fixes: dcfe378c81f7 ("lib: introduce support for page allocation tagging") would be the most appropriate.
I'll review all 3 fixes later today.
Thanks!
Re: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()
Quoted 23 lines · Suren Baghdasaryan
Quoted 9 lines · Andrew Morton
Quoted 6 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:
Add
pgalloc_tag_copy()to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump intocompaction_alloc, e.g.,# echo 1 >/proc/sys/vm/compact_memory # grep compaction_alloc /proc/allocinfoQuoted 8 lines
Quoted 6 lines
Before this patch:
132968448 32463mm/compaction.c:1880func:compaction_allocAfter this patch:
0 0mm/compaction.c:1880func:compaction_allocI'm thinking that [2/3] should be backported?
Yes, should be CC'ed to stable #6.10
Quoted 1 line
Quoted 6 lines
Quoted 1 line
And possibly this one, but for that we should identify a Fixes:, please.
I think for this one Fixes:
dcfe378c81f7("lib: introduce support for page allocation tagging") would be the most appropriate.I'll review all 3 fixes later today.
Looks good to me. Both [2/3] and [3/3] should be sent to stable 6.10.
Quoted 1 line
Thanks!
Quoted 3 lines · Yu Zhao
Fixes 22d407b164ff ("lib: add allocation tagging support for memory allocation profiling")Signed-off-by Yu Zhao <[email protected]>
Quoted 20 lines
--- include/linux/alloc_tag.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)include/linux/alloc_tag.h +1
-2@@ -70,7 +70,7 @@ static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)/* * When percpu variables are required to be defined as weak, static percpu * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION). -* Instead we will accound all module allocations to a single counter. +* Instead we will account all module allocations to a single counter. */ DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); --2.46.0.469.g59c65b2a67-goog
Quoted 6 lines · Yu Zhao
Fixes 22d407b164ff ("lib: add allocation tagging support for memory allocation profiling")Signed-off-by Yu Zhao <[email protected]>
Thanks.