Skip to contentSkip to conversation

mm/codetag: fix a typo

PATCH Superseded Yu Zhao 4 participants 9 messages Sep 3, 2024 – Sep 6, 2024

Subsystems (7)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Yu Zhao

author
#0 lore raw

PATCH 1/3 mm/codetag: fix a typo

Fixes 22d407b164ff ("lib: add allocation tagging support for memory allocation profiling")
---
 include/linux/alloc_tag.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/alloc_tag.h +1 -1
index 8c61ccd161ba..896491d9ebe8 100644
@@ -70,7 +70,7 @@ static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
 /*
  * When percpu variables are required to be defined as weak, static percpu
  * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
- * Instead we will accound all module allocations to a single counter.
+ * Instead we will account all module allocations to a single counter.
  */
 DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
 
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
author
#1 lore raw

PATCH 2/3 mm/codetag: fix pgalloc_tag_split()

Only tag the new head pages when splitting one large folio to multiple ones of a lower order. Tagging tail pages can cause imbalanced "calls" counters, since only head pages are untagged by pgalloc_tag_sub() and reference counts on tail pages are leaked, e.g.,

  # echo 2048kB >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote_size
  # echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
  # time echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote
  # grep alloc_gigantic_folio /proc/allocinfo

Before this patch:
0 549427200 mm/hugetlb.c:1549 func:alloc_gigantic_folio

  real  0m2.057s
  user  0m0.000s
  sys   0m2.051s

After this patch:
0 0 mm/hugetlb.c:1549 func:alloc_gigantic_folio

  real  0m1.711s
  user  0m0.000s
  sys   0m1.704s

Not tagging tail pages also improves the splitting time, e.g., by about 15% when demoting 1GB hugeTLB folios to 2MB ones, as shown above.

Fixes be25d1d4e822 ("mm: create new codetag references during page splitting")
---
 include/linux/mm.h          | 30 ++++++++++++++++++++++++++++++
 include/linux/pgalloc_tag.h | 31 -------------------------------
 mm/huge_memory.c            |  2 +-
 mm/hugetlb.c                |  2 +-
 mm/page_alloc.c             |  4 ++--
 5 files changed, 34 insertions(+), 35 deletions(-)
include/linux/mm.h +30 -0
index b31d4bdd65ad..a07e93adb8ad 100644
@@ -4137,4 +4137,34 @@ void vma_pgtable_walk_end(struct vm_area_struct *vma);
 
 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);
 
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
+{
+	int i;
+	struct alloc_tag *tag;
+	unsigned int nr_pages = 1 << new_order;
+
+	if (!mem_alloc_profiling_enabled())
+		return;
+
+	tag = pgalloc_tag_get(&folio->page);
+	if (!tag)
+		return;
+
+	for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
+		union codetag_ref *ref = get_page_tag_ref(folio_page(folio, i));
+
+		if (ref) {
+			/* Set new reference to point to the original tag */
+			alloc_tag_ref_set(ref, tag);
+			put_page_tag_ref(ref);
+		}
+	}
+}
+#else /* !CONFIG_MEM_ALLOC_PROFILING */
+static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
+{
+}
+#endif /* CONFIG_MEM_ALLOC_PROFILING */
+
 #endif /* _LINUX_MM_H */
include/linux/pgalloc_tag.h +0 -31
index 207f0c83c8e9..59a3deb792a8 100644
@@ -80,36 +80,6 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
 	}
 }
 
-static inline void pgalloc_tag_split(struct page *page, unsigned int nr)
-{
-	int i;
-	struct page_ext *first_page_ext;
-	struct page_ext *page_ext;
-	union codetag_ref *ref;
-	struct alloc_tag *tag;
-
-	if (!mem_alloc_profiling_enabled())
-		return;
-
-	first_page_ext = page_ext = page_ext_get(page);
-	if (unlikely(!page_ext))
-		return;
-
-	ref = codetag_ref_from_page_ext(page_ext);
-	if (!ref->ct)
-		goto out;
-
-	tag = ct_to_alloc_tag(ref->ct);
-	page_ext = page_ext_next(page_ext);
-	for (i = 1; i < nr; i++) {
-		/* Set new reference to point to the original tag */
-		alloc_tag_ref_set(codetag_ref_from_page_ext(page_ext), tag);
-		page_ext = page_ext_next(page_ext);
-	}
-out:
-	page_ext_put(first_page_ext);
-}
-
 static inline struct alloc_tag *pgalloc_tag_get(struct page *page)
 {
 	struct alloc_tag *tag = NULL;
@@ -142,7 +112,6 @@ static inline void clear_page_tag_ref(struct page *page) {}
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
 				   unsigned int nr) {}
 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
-static inline void pgalloc_tag_split(struct page *page, unsigned int nr) {}
 static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }
 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
 
mm/huge_memory.c +1 -1
index 0993dfe9ae94..aa8a4c938ba9 100644
@@ -3244,7 +3244,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
 	/* Caller disabled irqs, so they are still disabled here */
 
 	split_page_owner(head, order, new_order);
-	pgalloc_tag_split(head, 1 << order);
+	pgalloc_tag_split(folio, order, new_order);
 
 	/* See comment in __split_huge_page_tail() */
 	if (folio_test_anon(folio)) {
mm/hugetlb.c +1 -1
index 3faf5aad142d..a8624c07d8bf 100644
@@ -3778,7 +3778,7 @@ static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,
 		list_del(&folio->lru);
 
 		split_page_owner(&folio->page, huge_page_order(src), huge_page_order(dst));
-		pgalloc_tag_split(&folio->page, 1 <<  huge_page_order(src));
+		pgalloc_tag_split(folio, huge_page_order(src), huge_page_order(dst));
 
 		for (i = 0; i < pages_per_huge_page(src); i += pages_per_huge_page(dst)) {
 			struct page *page = folio_page(folio, i);
mm/page_alloc.c +2 -2
index c242d61fc4fd..13ce8e8899ed 100644
@@ -2822,7 +2822,7 @@ void split_page(struct page *page, unsigned int order)
 	for (i = 1; i < (1 << order); i++)
 		set_page_refcounted(page + i);
 	split_page_owner(page, order, 0);
-	pgalloc_tag_split(page, 1 << order);
+	pgalloc_tag_split(page_folio(page), order, 0);
 	split_page_memcg(page, order, 0);
 }
 EXPORT_SYMBOL_GPL(split_page);
@@ -5020,7 +5020,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order,
 		struct page *last = page + nr;
 
 		split_page_owner(page, order, 0);
-		pgalloc_tag_split(page, 1 << order);
+		pgalloc_tag_split(page_folio(page), order, 0);
 		split_page_memcg(page, order, 0);
 		while (page < --last)
 			set_page_refcounted(last);
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
maintainer Acked-by on v1 2/3
#2 lore raw

Re: [PATCH mm-unstable v1 2/3] mm/codetag: fix pgalloc_tag_split()

Quoted 131 lines · Yu Zhao

Only tag the new head pages when splitting one large folio to multiple ones of a lower order. Tagging tail pages can cause imbalanced "calls" counters, since only head pages are untagged by pgalloc_tag_sub() and reference counts on tail pages are leaked, e.g.,

  # echo 2048kB >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote_size
  # echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
  # time echo 700 >/sys/kernel/mm/hugepages/hugepages-1048576kB/demote
  # grep alloc_gigantic_folio /proc/allocinfo

Before this patch:
0 549427200 mm/hugetlb.c:1549 func:alloc_gigantic_folio

  real  0m2.057s
  user  0m0.000s
  sys   0m2.051s

After this patch:
0 0 mm/hugetlb.c:1549 func:alloc_gigantic_folio

  real  0m1.711s
  user  0m0.000s
  sys   0m1.704s

Not tagging tail pages also improves the splitting time, e.g., by about 15% when demoting 1GB hugeTLB folios to 2MB ones, as shown above.

Fixes be25d1d4e822 ("mm: create new codetag references during page splitting")
---
 include/linux/mm.h          | 30 ++++++++++++++++++++++++++++++
 include/linux/pgalloc_tag.h | 31 -------------------------------
 mm/huge_memory.c            |  2 +-
 mm/hugetlb.c                |  2 +-
 mm/page_alloc.c             |  4 ++--
 5 files changed, 34 insertions(+), 35 deletions(-)
include/linux/mm.h +30 -0
index b31d4bdd65ad..a07e93adb8ad 100644
@@ -4137,4 +4137,34 @@ void vma_pgtable_walk_end(struct vm_area_struct *vma);
 
 int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);
 
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
+{
+       int i;
+       struct alloc_tag *tag;
+       unsigned int nr_pages = 1 << new_order;
+
+       if (!mem_alloc_profiling_enabled())
+               return;
+
+       tag = pgalloc_tag_get(&folio->page);
+       if (!tag)
+               return;
+
+       for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
+               union codetag_ref *ref = get_page_tag_ref(folio_page(folio, i));
+
+               if (ref) {
+                       /* Set new reference to point to the original tag */
+                       alloc_tag_ref_set(ref, tag);
+                       put_page_tag_ref(ref);
+               }
+       }
+}
+#else /* !CONFIG_MEM_ALLOC_PROFILING */
+static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
+{
+}
+#endif /* CONFIG_MEM_ALLOC_PROFILING */
+
 #endif /* _LINUX_MM_H */
include/linux/pgalloc_tag.h +0 -31
index 207f0c83c8e9..59a3deb792a8 100644
@@ -80,36 +80,6 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
        }
 }
 
-static inline void pgalloc_tag_split(struct page *page, unsigned int nr)
-{
-       int i;
-       struct page_ext *first_page_ext;
-       struct page_ext *page_ext;
-       union codetag_ref *ref;
-       struct alloc_tag *tag;
-
-       if (!mem_alloc_profiling_enabled())
-               return;
-
-       first_page_ext = page_ext = page_ext_get(page);
-       if (unlikely(!page_ext))
-               return;
-
-       ref = codetag_ref_from_page_ext(page_ext);
-       if (!ref->ct)
-               goto out;
-
-       tag = ct_to_alloc_tag(ref->ct);
-       page_ext = page_ext_next(page_ext);
-       for (i = 1; i < nr; i++) {
-               /* Set new reference to point to the original tag */
-               alloc_tag_ref_set(codetag_ref_from_page_ext(page_ext), tag);
-               page_ext = page_ext_next(page_ext);
-       }
-out:
-       page_ext_put(first_page_ext);
-}
-
 static inline struct alloc_tag *pgalloc_tag_get(struct page *page)
 {
        struct alloc_tag *tag = NULL;
@@ -142,7 +112,6 @@ static inline void clear_page_tag_ref(struct page *page) {}
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
                                   unsigned int nr) {}
 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
-static inline void pgalloc_tag_split(struct page *page, unsigned int nr) {}
 static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }
 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
 
mm/huge_memory.c +0 -0
index 0993dfe9ae94..aa8a4c938ba9 100644

mm/huge_memory.c static void __split_huge_page(struct page *page, struct list_head *list, · line 3247

Quoted 5 lines
mm/huge_memory.c +1 -1
@@ -3244,7 +3244,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
        /* Caller disabled irqs, so they are still disabled here */
 
        split_page_owner(head, order, new_order);
-       pgalloc_tag_split(head, 1 << order);
+       pgalloc_tag_split(folio, order, new_order);

Looks like here and in the next diff you are fixing an additional bug. I was assuming that we are always splitting into order-0 pages, which is not the case anymore. It's worth mentioning that in the changelog. With that added:

Quoted 41 lines
mm/huge_memory.c +0 -0
        /* See comment in __split_huge_page_tail() */
        if (folio_test_anon(folio)) {
mm/hugetlb.c +1 -1
index 3faf5aad142d..a8624c07d8bf 100644
@@ -3778,7 +3778,7 @@ static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,
                list_del(&folio->lru);
 
                split_page_owner(&folio->page, huge_page_order(src), huge_page_order(dst));
-               pgalloc_tag_split(&folio->page, 1 <<  huge_page_order(src));
+               pgalloc_tag_split(folio, huge_page_order(src), huge_page_order(dst));
 
                for (i = 0; i < pages_per_huge_page(src); i += pages_per_huge_page(dst)) {
                        struct page *page = folio_page(folio, i);
mm/page_alloc.c +2 -3
index c242d61fc4fd..13ce8e8899ed 100644
@@ -2822,7 +2822,7 @@ void split_page(struct page *page, unsigned int order)
        for (i = 1; i < (1 << order); i++)
                set_page_refcounted(page + i);
        split_page_owner(page, order, 0);
-       pgalloc_tag_split(page, 1 << order);
+       pgalloc_tag_split(page_folio(page), order, 0);
        split_page_memcg(page, order, 0);
 }
 EXPORT_SYMBOL_GPL(split_page);
@@ -5020,7 +5020,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order,
                struct page *last = page + nr;
 
                split_page_owner(page, order, 0);
-               pgalloc_tag_split(page, 1 << order);
+               pgalloc_tag_split(page_folio(page), order, 0);
                split_page_memcg(page, order, 0);
                while (page < --last)
                        set_page_refcounted(last);
--

2.46.0.469.g59c65b2a67-goog

author
#3 lore raw

PATCH 3/3 mm/codetag: add pgalloc_tag_copy()

In reply to Yu Zhao

Add pgalloc_tag_copy() to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump into compaction_alloc, e.g.,

  # echo 1 >/proc/sys/vm/compact_memory
  # grep compaction_alloc /proc/allocinfo

Before this patch:
132968448 32463 mm/compaction.c:1880 func:compaction_alloc

After this patch:
0 0 mm/compaction.c:1880 func:compaction_alloc

---
 include/linux/alloc_tag.h | 24 ++++++++++--------------
 include/linux/mm.h        | 25 +++++++++++++++++++++++++
 mm/migrate.c              |  1 +
 3 files changed, 36 insertions(+), 14 deletions(-)
include/linux/alloc_tag.h +10 -14
index 896491d9ebe8..1f0a9ff23a2c 100644
@@ -137,7 +137,16 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
 /* Caller should verify both ref and tag to be valid */
 static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
 {
+	alloc_tag_add_check(ref, tag);
+	if (!ref || !tag)
+		return;
+
 	ref->ct = &tag->ct;
+}
+
+static inline void alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
+{
+	__alloc_tag_ref_set(ref, tag);
 	/*
 	 * We need in increment the call counter every time we have a new
 	 * allocation or when we split a large allocation into smaller ones.
@@ -147,22 +156,9 @@ static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag
 	this_cpu_inc(tag->counters->calls);
 }
 
-static inline void alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
-{
-	alloc_tag_add_check(ref, tag);
-	if (!ref || !tag)
-		return;
-
-	__alloc_tag_ref_set(ref, tag);
-}
-
 static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
 {
-	alloc_tag_add_check(ref, tag);
-	if (!ref || !tag)
-		return;
-
-	__alloc_tag_ref_set(ref, tag);
+	alloc_tag_ref_set(ref, tag);
 	this_cpu_add(tag->counters->bytes, bytes);
 }
 
include/linux/mm.h +25 -0
index a07e93adb8ad..1b98d843a5e9 100644
@@ -4161,10 +4161,35 @@ static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new
 		}
 	}
 }
+
+static inline void pgalloc_tag_copy(struct folio *new, struct folio *old)
+{
+	struct alloc_tag *tag;
+	union codetag_ref *ref;
+
+	tag = pgalloc_tag_get(&old->page);
+	if (!tag)
+		return;
+
+	ref = get_page_tag_ref(&new->page);
+	if (!ref)
+		return;
+
+	/* Clear the old ref to the original allocation site. */
+	clear_page_tag_ref(&old->page);
+	/* Decrement the counters of the tag on get_new_folio. */
+	alloc_tag_sub(ref, folio_nr_pages(new));
+	__alloc_tag_ref_set(ref, tag);
+	put_page_tag_ref(ref);
+}
 #else /* !CONFIG_MEM_ALLOC_PROFILING */
 static inline void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
 {
 }
+
+static inline void pgalloc_tag_copy(struct folio *new, struct folio *old)
+{
+}
 #endif /* CONFIG_MEM_ALLOC_PROFILING */
 
 #endif /* _LINUX_MM_H */
mm/migrate.c +1 -0
index 35cc9d35064b..0b24021d5fee 100644
@@ -750,6 +750,7 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
 		folio_set_readahead(newfolio);
 
 	folio_copy_owner(newfolio, folio);
+	pgalloc_tag_copy(newfolio, folio);
 
 	mem_cgroup_migrate(folio, newfolio);
 }
Signature · 1 line
2.46.0.469.g59c65b2a67-goog
maintainer
#4 lore raw

Re: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()

Quoted 12 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:

Add pgalloc_tag_copy() to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump into compaction_alloc, e.g.,

  # echo 1 >/proc/sys/vm/compact_memory
  # grep compaction_alloc /proc/allocinfo

Before this patch:
132968448 32463 mm/compaction.c:1880 func:compaction_alloc

After this patch:
0 0 mm/compaction.c:1880 func:compaction_alloc

I'm thinking that [2/3] should be backported?

And possibly this one, but for that we should identify a Fixes:, please.

#5 lore raw

Re: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()

Quoted 9 lines · Andrew Morton
Quoted 6 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:

Add pgalloc_tag_copy() to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump into compaction_alloc, e.g.,

  # echo 1 >/proc/sys/vm/compact_memory
  # grep compaction_alloc /proc/allocinfo
Quoted 8 lines
Quoted 6 lines

Before this patch:
132968448 32463 mm/compaction.c:1880 func:compaction_alloc

After this patch:
0 0 mm/compaction.c:1880 func:compaction_alloc

I'm thinking that [2/3] should be backported?

Yes, should be CC'ed to stable #6.10

Quoted 2 lines

And possibly this one, but for that we should identify a Fixes:, please.

I think for this one Fixes: dcfe378c81f7 ("lib: introduce support for page allocation tagging") would be the most appropriate.

I'll review all 3 fixes later today.
Thanks!

maintainer Acked-by
#6 lore raw

Re: [PATCH mm-unstable v1 3/3] mm/codetag: add pgalloc_tag_copy()

Quoted 23 lines · Suren Baghdasaryan
Quoted 9 lines · Andrew Morton
Quoted 6 lines · On Tue, 3 Sep 2024 15:36:49 -0600 Yu Zhao <[email protected]> wrote:

Add pgalloc_tag_copy() to transfer the codetag from the old folio to the new one during migration. This makes original allocation sites persist cross migration rather than lump into compaction_alloc, e.g.,

  # echo 1 >/proc/sys/vm/compact_memory
  # grep compaction_alloc /proc/allocinfo
Quoted 8 lines
Quoted 6 lines

Before this patch:
132968448 32463 mm/compaction.c:1880 func:compaction_alloc

After this patch:
0 0 mm/compaction.c:1880 func:compaction_alloc

I'm thinking that [2/3] should be backported?

Yes, should be CC'ed to stable #6.10

Quoted 1 line
Quoted 6 lines
Quoted 1 line

And possibly this one, but for that we should identify a Fixes:, please.

I think for this one Fixes: dcfe378c81f7 ("lib: introduce support for page allocation tagging") would be the most appropriate.

I'll review all 3 fixes later today.

Looks good to me. Both [2/3] and [3/3] should be sent to stable 6.10.

Quoted 1 line

Thanks!

maintainer Acked-by on v1 1/3
#7 lore raw
In reply to Yu Zhao
Quoted 3 lines · Yu Zhao
Fixes 22d407b164ff ("lib: add allocation tagging support for memory allocation profiling")
Quoted 20 lines
---
 include/linux/alloc_tag.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/alloc_tag.h +1 -2
index 8c61ccd161ba..896491d9ebe8 100644
@@ -70,7 +70,7 @@ static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
 /*
  * When percpu variables are required to be defined as weak, static percpu
  * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
- * Instead we will accound all module allocations to a single counter.
+ * Instead we will account all module allocations to a single counter.
  */
 DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
 
--

2.46.0.469.g59c65b2a67-goog