mm/rmap: separate hugetlb rmap handling
Subsystems (5)
cover mm/rmap: separate hugetlb rmap handling
Let's just cleanly separate hugetlb rmap handling from "ordinary" rmap handling, like we already do when adding anon hugetlb folios. We have hugetlb special-casing/checks in the callers in all cases either way already in place: it doesn't make too much sense to call generic-looking functions that end up doing hugetlb specific things from hugetlb special-cases.
This now also means that we won't run into "folio_test_pmd_mappable()" cases for hugetlb, which doesn't make too much sense with gigantic hugetlb folios that are in fact currently only PUD-mappable. Or having a folio_add_file_rmap() function that looks like it clould be used by hugetlb code, but really can't.
This is a stanalone cleanup, but it gets more relevant when adding more rmap batching (we cannot map parts of a hugetlb folio) or changing the way we handle partially-mappable folios as in [1], whereby we'd have to add more hugetlb special casing to keep hugetlb working as is.
If ever something about hugetlb changes that makes them actually partially-mappable folios, we can look into cleanly merging all code paths, not just some.
[1] https://lkml.kernel.org/r/[email protected]
David Hildenbrand (5):
mm/rmap: rename hugepage_add* to hugetlb_add* mm/rmap: introduce and use hugetlb_remove_rmap() mm/rmap: introduce and use hugetlb_add_file_rmap() mm/rmap: introduce and use hugetlb_try_dup_anon_rmap() mm/rmap: add hugetlb sanity checks
include/linux/mm.h | 12 +++++++++--- include/linux/rmap.h | 37 +++++++++++++++++++++++++++++++++++-- mm/hugetlb.c | 21 ++++++++++----------- mm/migrate.c | 6 +++--- mm/rmap.c | 31 ++++++++++++++++++------------- 5 files changed, 75 insertions(+), 32 deletions(-)
base-commit: b85ea95d086471afb4ad062012a4d73cd328fa86
Signature · 1 line
2.41.0
PATCH 1/5 mm/rmap: rename hugepage_add* to hugetlb_add*
Let's just call it "hugetlb_".
Yes, it's all already inconsistent and confusing because we have a lot of "hugepage_" functions for legacy reasons. But "hugetlb" cannot possibly be confused with transparent huge pages, and it matches "hugetlb.c" and "folio_test_hugetlb()". So let's minimize confusion in rmap code.
--- include/linux/rmap.h | 4 ++-- mm/hugetlb.c | 8 ++++---- mm/migrate.c | 4 ++-- mm/rmap.c | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-)
include/linux/rmap.h +2 -2
@@ -203,9 +203,9 @@ void folio_add_file_rmap_range(struct folio *, struct page *, unsigned int nr,void page_remove_rmap (struct page *,struct vm_area_struct *,bool compound); -void hugepage_add_anon_rmap (struct folio *,struct vm_area_struct *, +void hugetlb_add_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address,rmap_t flags); -void hugepage_add_new_anon_rmap (struct folio *,struct vm_area_struct *, +void hugetlb_add_new_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address);static inline void __page_dup_rmap (struct page *page,bool compound)
mm/hugetlb.c +4 -4
@@ -5278,7 +5278,7 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addpte_t newpte =make_huge_pte (vma, &new_folio->page,1 );__folio_mark_uptodate (new_folio); -hugepage_add_new_anon_rmap (new_folio, vma, addr); +hugetlb_add_new_anon_rmap (new_folio, vma, addr);if (userfaultfd_wp (vma) &&huge_pte_uffd_wp (old)) newpte =huge_pte_mkuffd_wp (newpte);set_huge_pte_at (vma->vm_mm, addr, ptep, newpte, sz);
@@ -5981,7 +5981,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,/* Break COW or unshare */ huge_ptep_clear_flush (vma, haddr, ptep);page_remove_rmap (&old_folio->page, vma, true); -hugepage_add_new_anon_rmap (new_folio, vma, haddr); +hugetlb_add_new_anon_rmap (new_folio, vma, haddr);if (huge_pte_uffd_wp (pte)) newpte =huge_pte_mkuffd_wp (newpte);set_huge_pte_at (mm, haddr, ptep, newpte,huge_page_size (h));
@@ -6270,7 +6270,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, goto backout;if (anon_rmap) -hugepage_add_new_anon_rmap (folio, vma, haddr); +hugetlb_add_new_anon_rmap (folio, vma, haddr);else page_dup_file_rmap (&folio->page, true); new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
@@ -6725,7 +6725,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,if (folio_in_pagecache)page_dup_file_rmap (&folio->page, true);else -hugepage_add_new_anon_rmap (folio, dst_vma, dst_addr); +hugetlb_add_new_anon_rmap (folio, dst_vma, dst_addr);/* * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
mm/migrate.c +2 -2
@@ -249,8 +249,8 @@ static bool remove_migration_pte(struct folio *folio, pte =arch_make_huge_pte (pte, shift, vma->vm_flags);if (folio_test_anon (folio)) -hugepage_add_anon_rmap (folio, vma, pvmw.address, - rmap_flags); +hugetlb_add_anon_rmap (folio, vma, pvmw.address, + rmap_flags);else page_dup_file_rmap (new, true); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte,
mm/rmap.c +4 -4
@@ -2583,8 +2583,8 @@ void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)* * RMAP_COMPOUND is ignored. */ -void hugepage_add_anon_rmap (struct folio *folio,struct vm_area_struct *vma, -unsigned long address,rmap_t flags) +void hugetlb_add_anon_rmap (struct folio *folio,struct vm_area_struct *vma, +unsigned long address,rmap_t flags) {VM_WARN_ON_FOLIO (!folio_test_anon (folio), folio);
@@ -2595,8 +2595,8 @@ void hugepage_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, PageAnonExclusive(&folio->page), folio); } -void hugepage_add_new_anon_rmap (struct folio *folio, -struct vm_area_struct *vma,unsigned long address) +void hugetlb_add_new_anon_rmap (struct folio *folio, +struct vm_area_struct *vma,unsigned long address) {BUG_ON (address < vma->vm_start || address >= vma->vm_end);/* increment count (starts at -1) */
Signature · 1 line
2.41.0
Re: [PATCH v1 1/5] mm/rmap: rename hugepage_add* to hugetlb_add*
Quoted 12 lines · David Hildenbrand
Let's just call it "
hugetlb_".Yes, it's all already inconsistent and confusing because we have a lot of "
hugepage_" functions for legacy reasons. But "hugetlb" cannot possibly be confused with transparent huge pages, and it matches "hugetlb.c" and "folio_test_hugetlb()". So let's minimize confusion in rmap code.Signed-off-by David Hildenbrand <[email protected]>
Thanks.
PATCH 2/5 mm/rmap: introduce and use hugetlb_remove_rmap()
In reply to David Hildenbrandhugetlb rmap handling differs quite a lot from "ordinary" rmap code. We don't want this hugetlb special-casing in the rmap functions, as we're special casing the callers already. Let's simply use a separate function for hugetlb.
Let's introduce and use hugetlb_remove_rmap() and remove the hugetlb code from page_remove_rmap(). This effectively removes one check on the small-folio path as well.
While this is a cleanup, this will also make it easier to change rmap handling for partially-mappable folios.
Note: all possible candidates that need care are page_remove_rmap() that
pass compound=true.
--- include/linux/rmap.h | 5 +++++ mm/hugetlb.c | 4 ++-- mm/rmap.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-)
include/linux/rmap.h +5 -0
@@ -208,6 +208,11 @@ void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,void hugetlb_add_new_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address); +static inline void hugetlb_remove_rmap (struct folio *folio) +{ +atomic_dec (&folio->_entire_mapcount); +} +static inline void __page_dup_rmap (struct page *page,bool compound) {if (compound) {
mm/hugetlb.c +2 -2
@@ -5669,7 +5669,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,make_pte_marker (PTE_MARKER_UFFD_WP ), sz);hugetlb_count_sub (pages_per_huge_page (h), mm); -page_remove_rmap (page, vma, true); +hugetlb_remove_rmap (page_folio (page));spin_unlock (ptl);tlb_remove_page_size (tlb, page,huge_page_size (h));
@@ -5980,7 +5980,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,/* Break COW or unshare */ huge_ptep_clear_flush (vma, haddr, ptep); -page_remove_rmap (&old_folio->page, vma, true); +hugetlb_remove_rmap (old_folio);hugetlb_add_new_anon_rmap (new_folio, vma, haddr);if (huge_pte_uffd_wp (pte)) newpte =huge_pte_mkuffd_wp (newpte);
mm/rmap.c +8 -9
@@ -1440,13 +1440,6 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma,VM_BUG_ON_PAGE (compound && !PageHead (page), page); -/* Hugetlb pages are not counted in NR_*MAPPED */ -if (unlikely (folio_test_hugetlb (folio))) { -/* hugetlb pages are always mapped with pmds */ -atomic_dec (&folio->_entire_mapcount); -return ; - } -/* Is page being unmapped by PTE? Is this its last map to be removed? */ if (likely (!compound)) { last =atomic_add_negative (-1 , &page->_mapcount);
@@ -1804,7 +1797,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,dec_mm_counter (mm,mm_counter_file (&folio->page)); } discard: -page_remove_rmap (subpage, vma,folio_test_hugetlb (folio)); +if (unlikely (folio_test_hugetlb (folio))) +hugetlb_remove_rmap (folio); +else +page_remove_rmap (subpage, vma, false);if (vma->vm_flags &VM_LOCKED )mlock_drain_local ();folio_put (folio);
@@ -2157,7 +2153,10 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,*/ } -page_remove_rmap (subpage, vma,folio_test_hugetlb (folio)); +if (unlikely (folio_test_hugetlb (folio))) +hugetlb_remove_rmap (folio); +else +page_remove_rmap (subpage, vma, false);if (vma->vm_flags &VM_LOCKED )mlock_drain_local ();folio_put (folio);
Signature · 1 line
2.41.0
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 4 lines · David Hildenbrand
hugetlb rmap handling differs quite a lot from "ordinary" rmap code. We don't want this hugetlb special-casing in the rmap functions, as we're special casing the callers already. Let's simply use a separate function for hugetlb.
We were special casing some, not all..
And IIUC the suggestion from the community is we reduce that "special casing", instead of adding more? To be explicit below..
Quoted 61 lines
Let's introduce and use
hugetlb_remove_rmap()and remove the hugetlb code frompage_remove_rmap(). This effectively removes one check on the small-folio path as well.While this is a cleanup, this will also make it easier to change rmap handling for partially-mappable folios.
Note: all possible candidates that need care are
page_remove_rmap()that
pass compound=true.Signed-off-by David Hildenbrand <[email protected]>--- include/linux/rmap.h | 5 +++++ mm/hugetlb.c | 4 ++-- mm/rmap.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-)include/linux/rmap.h +5
-0@@ -208,6 +208,11 @@ void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,void hugetlb_add_new_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address); +static inline void hugetlb_remove_rmap (struct folio *folio) +{ +atomic_dec (&folio->_entire_mapcount); +} +static inline void __page_dup_rmap (struct page *page,bool compound) {if (compound) {mm/hugetlb.c +2
-2@@ -5669,7 +5669,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,make_pte_marker (PTE_MARKER_UFFD_WP ), sz);hugetlb_count_sub (pages_per_huge_page (h), mm); -page_remove_rmap (page, vma, true); +hugetlb_remove_rmap (page_folio (page));spin_unlock (ptl);tlb_remove_page_size (tlb, page,huge_page_size (h));@@ -5980,7 +5980,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,/* Break COW or unshare */ huge_ptep_clear_flush (vma, haddr, ptep); -page_remove_rmap (&old_folio->page, vma, true); +hugetlb_remove_rmap (old_folio);hugetlb_add_new_anon_rmap (new_folio, vma, haddr);if (huge_pte_uffd_wp (pte)) newpte =huge_pte_mkuffd_wp (newpte);mm/rmap.c +0
-0
mm/rmap.c void page_remove_rmap(struct page *page, struct vm_area_struct *vma, · line 1448
Quoted 9 lines
mm/rmap.c +0
-6@@ -1440,13 +1440,6 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma,VM_BUG_ON_PAGE (compound && !PageHead (page), page); -/* Hugetlb pages are not counted in NR_*MAPPED */ -if (unlikely (folio_test_hugetlb (folio))) { -/* hugetlb pages are always mapped with pmds */ -atomic_dec (&folio->_entire_mapcount); -return ; - }
Fundamentally in the ideal world when hugetlb can be merged into generic mm, I'd imagine that as dropping here, meanwhile...
mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 1803
Quoted 13 lines
mm/rmap.c +4
-2-/* Is page being unmapped by PTE? Is this its last map to be removed? */ if (likely (!compound)) { last =atomic_add_negative (-1 , &page->_mapcount);@@ -1804,7 +1797,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,dec_mm_counter (mm,mm_counter_file (&folio->page)); } discard: -page_remove_rmap (subpage, vma,folio_test_hugetlb (folio)); +if (unlikely (folio_test_hugetlb (folio))) +hugetlb_remove_rmap (folio); +else +page_remove_rmap (subpage, vma, false);
... rather than moving hugetlb_* handlings even upper the stack, we should hopefully be able to keep this one as a generic api.
I worry this patch is adding more hugetlb-specific paths even onto higher call stacks so it's harder to generalize, going the other way round to what we wanted per previous discussions.
Said that, indeed I read mostly nothing yet with the recent rmap patches, so I may miss some important goal here.
Quoted 19 lines
mm/rmap.c +4
-1if (vma->vm_flags &VM_LOCKED )mlock_drain_local ();folio_put (folio);@@ -2157,7 +2153,10 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,*/ } -page_remove_rmap (subpage, vma,folio_test_hugetlb (folio)); +if (unlikely (folio_test_hugetlb (folio))) +hugetlb_remove_rmap (folio); +else +page_remove_rmap (subpage, vma, false);if (vma->vm_flags &VM_LOCKED )mlock_drain_local ();folio_put (folio);Signature · 1 line
2.41.0
Thanks,
Signature · 1 line
Peter Xu
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 10 lines · Peter Xu
Quoted 4 lines · David Hildenbrand
hugetlb rmap handling differs quite a lot from "ordinary" rmap code. We don't want this hugetlb special-casing in the rmap functions, as we're special casing the callers already. Let's simply use a separate function for hugetlb.
We were special casing some, not all..
And IIUC the suggestion from the community is we reduce that "special casing", instead of adding more? To be explicit below..
Quoting from the cover letter:
"We have hugetlb special-casing/checks in the callers in all cases either way already in place: it doesn't make too much sense to call generic-looking functions that end up doing hugetlb specific things from hugetlb special-cases."
[...]
Quoted 5 lines
Quoted 5 lines
+++
b/mm/rmap.cmm/rmap.c +0
-0@@ -1440,13 +1440,6 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma,VM_BUG_ON_PAGE (compound && !PageHead (page), page);
Quoted 10 lines
mm/rmap.c void page_remove_rmap(struct page *page, struct vm_area_struct *vma, · line 1448
Quoted 6 lines
mm/rmap.c +0
-6-/* Hugetlb pages are not counted in NR_*MAPPED */ -if (unlikely (folio_test_hugetlb (folio))) { -/* hugetlb pages are always mapped with pmds */ -atomic_dec (&folio->_entire_mapcount); -return ; - }Fundamentally in the ideal world when hugetlb can be merged into generic mm, I'd imagine that as dropping here, meanwhile...
Quoting from the cover letter:
"If ever something about hugetlb changes that makes them actually partially-mappable folios, we can look into cleanly merging all code paths, not just some."
Quoted 17 lines
mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 1803
Quoted 13 lines
mm/rmap.c +4
-2-/* Is page being unmapped by PTE? Is this its last map to be removed? */ if (likely (!compound)) { last =atomic_add_negative (-1 , &page->_mapcount);@@ -1804,7 +1797,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,dec_mm_counter (mm,mm_counter_file (&folio->page)); } discard: -page_remove_rmap (subpage, vma,folio_test_hugetlb (folio)); +if (unlikely (folio_test_hugetlb (folio))) +hugetlb_remove_rmap (folio); +else +page_remove_rmap (subpage, vma, false);... rather than moving hugetlb_* handlings even upper the stack, we should hopefully be able to keep this one as a generic api.
Quoted 6 lines
I worry this patch is adding more hugetlb-specific paths even onto higher call stacks so it's harder to generalize, going the other way round to what we wanted per previous discussions.
Said that, indeed I read mostly nothing yet with the recent rmap patches, so I may miss some important goal here.
Quoting from the cover letter:
"This is a stanalone cleanup, but it gets more relevant when adding more
rmap batching (we cannot map parts of a hugetlb folio) or changing the way we handle partially-mappable folios as in [1], whereby we'd have to add more hugetlb special casing to keep hugetlb working as is."
Thanks for the review. If you have strong feelings, please add an explicit nack. Otherwise, I'll move forward with this.
Signature · 3 lines
Cheers, David / dhildenb
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 6 lines · David Hildenbrand
Quoting from the cover letter:
"We have hugetlb special-casing/checks in the callers in all cases either way already in place: it doesn't make too much sense to call generic-looking functions that end up doing hugetlb specific things from hugetlb special-cases."
I'll take this one as an example: I think one goal (of my understanding of the mm community) is to make the generic looking functions keep being generic, dropping any function named as "*hugetlb*" if possible one day within that generic implementation. I said that in my previous reply.
Having that "*hugetlb*" code already in the code base may or may not be a good reason to further move it upward the stack.
Strong feelings? No, I don't have. I'm not knowledged enough to do so.
Thanks,
Signature · 1 line
Peter Xu
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 8 lines · Peter Xu
Quoted 6 lines · David Hildenbrand
Quoting from the cover letter:
"We have hugetlb special-casing/checks in the callers in all cases either way already in place: it doesn't make too much sense to call generic-looking functions that end up doing hugetlb specific things from hugetlb special-cases."
Quoted 4 lines
I'll take this one as an example: I think one goal (of my understanding of the mm community) is to make the generic looking functions keep being generic, dropping any function named as "*hugetlb*" if possible one day within that generic implementation. I said that in my previous reply.
Yes, and I am one of the people asking for that. However, only where it makes sense (e.g., like page table walking, GUP as you said), and only when it is actually unified.
I don't think that rmap handling or fault handling will ever be completely unified to that extreme, and it might also not be desirable. Just like we have separate paths for anon and file in areas where they are reasonable different.
What doesn't make sense is using patterns like:
page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
and then, inside page_remove_rmap() have an initial folio_test_hugetlb() check that does something completely different.
So each and everyone calling page_remove_rmap (and knowing that it's certainly not a hugetlb folio) has to run through that check.
Then, we have functions like page_add_file_rmap() that look like they can be used for hugetlb, but hugetlb is smart enough and only calls page_dup_file_rmap(), because it doesn't want to touch any file/anon counters. And to handle that we would have to add folio_test_hugetlb() inside there, which adds the same as above, trying to unify without unifying.
Once we're in the area of folio_add_file_rmap_range(), it all stops making sense, because there is no way we could possibly partially map a folio today. (and if we can in the future, we might still want separate handling, because most caller know with which pages they are dealing, below)
Last but not least, it's all inconsistent right now with hugetlb_add_anon_rmap()/hugetlb_add_new_anon_rmap() being there because they differ reasonably well from the "ordinary" counterparts.
Quoted 3 lines
Having that "*hugetlb*" code already in the code base may or may not be a good reason to further move it upward the stack.
If you see a path forward in the foreseeable future where we would have code that doesn't special-case hugetlb in rmap calling code already, I'd be interested in that.
hugetlb.c knows that it's dealing with hugetlb pages.
huge_memory.c knows that it's dealing with PMD-mapped thp.
memory.c knows that it it's dealing with PTE-mapped thp or small folios.
Only migrate.c (and e.g., try_to_unmap()) in rmap.c handle different types. But there is plenty of hugetlb special-casing in there that I don't really see going away.
Quoted 2 lines
Strong feelings? No, I don't have. I'm not knowledged enough to do so.
I'm sure you are, so I'm trusting your judgment :)
I don't think going in the other direction and e.g., removing hugetlb_add_anon_rmap / hugetlb_add_new_anon_rmap is making a unification that is not really reasonable. It will only make things slower and the individual functions more complicated.
Signature · 3 lines
Cheers, David / dhildenb
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 14 lines · David Hildenbrand
Quoted 8 lines · Peter Xu
Quoted 6 lines · David Hildenbrand
Quoting from the cover letter:
"We have hugetlb special-casing/checks in the callers in all cases either way already in place: it doesn't make too much sense to call generic-looking functions that end up doing hugetlb specific things from hugetlb special-cases."
Quoted 4 lines
I'll take this one as an example: I think one goal (of my understanding of the mm community) is to make the generic looking functions keep being generic, dropping any function named as "*hugetlb*" if possible one day within that generic implementation. I said that in my previous reply.
Quoted 8 lines
Yes, and I am one of the people asking for that. However, only where it makes sense (e.g., like page table walking, GUP as you said), and only when it is actually unified.
I don't think that rmap handling or fault handling will ever be completely unified to that extreme, and it might also not be desirable. Just like we have separate paths for anon and file in areas where they are reasonable different.
Yes I haven't check further in that direction, that'll be after the pgtable work for me if that can first move on smoothly, and it'll also depend on whether merging the pgtable changes would be good enough so that we can move on to consider small mappings for hugetlb, until then we need to settle a final mapcount solution for hugetlb.
Quoted 7 lines
What doesn't make sense is using patterns like:
page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));and then, inside
page_remove_rmap()have an initialfolio_test_hugetlb()check that does something completely different.
IIUC above "folio_test_hugetlb(folio)" pattern can become "false" for hugetlb, if we decided to do mapcount for small hugetlb mappings. If that happens, I think something like this patch _may_ need to be reverted again more or less. Or we start to copy some of page_remove_rmap() into the new hugetlb rmap api.
Quoted 3 lines
So each and everyone calling
page_remove_rmap(and knowing that it's certainly not a hugetlb folio) has to run through that check.
Note that right after this change applied, hugetlb already start to lose something in common with generic compound folios, where page_remove_rmap() had:
VM_BUG_ON_PAGE(compound && !PageHead(page), page);
That sanity check goes away immediately for hugetlb, which is still logically applicable.
Quoted 8 lines
Then, we have functions like
page_add_file_rmap()that look like they can be used for hugetlb, but hugetlb is smart enough and only callspage_dup_file_rmap(), because it doesn't want to touch any file/anon counters. And to handle that we would have to addfolio_test_hugetlb()inside there, which adds the same as above, trying to unify without unifying.
Quoted 8 lines
Once we're in the area of
folio_add_file_rmap_range(), it all stops making sense, because there is no way we could possibly partially map a folio today. (and if we can in the future, we might still want separate handling, because most caller know with which pages they are dealing, below)Last but not least, it's all inconsistent right now with hugetlb_add_anon_rmap()/hugetlb_add_new_anon_rmap() being there because they differ reasonably well from the "ordinary" counterparts.
Taking hugepage_add_new_anon_rmap() as example, IMHO they still share a lot of things with !hugetlb codes, and maybe they can already be cleaned up into something common for a large mapping:
void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
unsigned long address)
{
int nr;
VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
if (folio_is_hugetlb(folio)) {
folio_clear_hugetlb_restore_reserve(folio);
} else {
__folio_set_swapbacked(folio);
atomic_set(&folio->_nr_pages_mapped, COMPOUND_MAPPED);
nr = folio_nr_pages(folio);
__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr);
__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);
}
/* increment count (starts at -1) */
atomic_set(&folio->_entire_mapcount, 0);
__folio_set_anon(folio, vma, address, true);
SetPageAnonExclusive(&folio->page);
}For folio_add_file_rmap_range(): would it work if it just pass the hugetlb folio range into it? Would that make it much more difficult with the recent work on large folios from you or anyone?
Quoted 4 lines
I don't think going in the other direction and e.g., removing
hugetlb_add_anon_rmap/hugetlb_add_new_anon_rmapis making a unification that is not really reasonable. It will only make things slower and the individual functions more complicated.
IIUC so far the performance difference should be minimum on which helper to use.
As I mentioned, I sincerely don't know whether this patch is good or not as I don't know enough on everything around that is happening. It's just that I'll still think twice if to create hugetlb its own rmap API, because from the high level it's going the other way round to me. So I still want to raise this as a pure question.
Thanks,
Signature · 1 line
Peter Xu
Re: [PATCH v1 2/5] mm/rmap: introduce and use hugetlb_remove_rmap()
Quoted 4 lines
Quoted 4 lines
What doesn't make sense is using patterns like:
page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
Quoted 8 lines
Quoted 2 lines
and then, inside
page_remove_rmap()have an initialfolio_test_hugetlb()check that does something completely different.IIUC above "
folio_test_hugetlb(folio)" pattern can become "false" for hugetlb, if we decided to do mapcount for small hugetlb mappings. If that happens, I think something like this patch _may_ need to be reverted again more or less. Or we start to copy some ofpage_remove_rmap()into the new hugetlb rmap api.
Probably just extend it to a hugetlb rmap api if you want to do some batching. But who knows how that will actually look like. The migrate.c/rmap.c special casing for hugetlb when migrating/unmapping is quite severe already and this is just the tip of the iceberg.
Further, the question is if you really want to deal with subpage mapcounts like the !hugetlb variant does. Quite likely you don't want to do that if you can avoid it and just have a total mapcount.
Quoted 5 lines
Quoted 4 lines
So each and everyone calling
page_remove_rmap(and knowing that it's certainly not a hugetlb folio) has to run through that check.
Quoted 8 lines
Note that right after this change applied, hugetlb already start to lose something in common with generic compound folios, where
page_remove_rmap()had:VM_BUG_ON_PAGE(compound && !PageHead(page), page);That sanity check goes away immediately for hugetlb, which is still logically applicable.
Why? We're passing folios that the caller already properly prepared.
A folio by definition points to the head page.
Quoted 36 lines
Quoted 9 lines
Then, we have functions like
page_add_file_rmap()that look like they can be used for hugetlb, but hugetlb is smart enough and only callspage_dup_file_rmap(), because it doesn't want to touch any file/anon counters. And to handle that we would have to addfolio_test_hugetlb()inside there, which adds the same as above, trying to unify without unifying.Quoted 8 lines
Once we're in the area of
folio_add_file_rmap_range(), it all stops making sense, because there is no way we could possibly partially map a folio today. (and if we can in the future, we might still want separate handling, because most caller know with which pages they are dealing, below)Last but not least, it's all inconsistent right now with hugetlb_add_anon_rmap()/hugetlb_add_new_anon_rmap() being there because they differ reasonably well from the "ordinary" counterparts.
Taking
hugepage_add_new_anon_rmap()as example, IMHO they still share a lot of things with !hugetlb codes, and maybe they can already be cleaned up into something common for a large mapping:void folio_add_new_anon_rmap(struct folio *folio, struct
vm_area_struct*vma,
unsigned long address){ int nr; VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); if (folio_is_hugetlb(folio)) { folio_clear_hugetlb_restore_reserve(folio); } else { __folio_set_swapbacked(folio); atomic_set(&folio->_nr_pages_mapped, COMPOUND_MAPPED); nr = folio_nr_pages(folio); __lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr);
Quoted 8 lines
__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);
}
/* increment count (starts at -1) */
atomic_set(&folio->_entire_mapcount, 0);
__folio_set_anon(folio, vma, address, true);
SetPageAnonExclusive(&folio->page);
}Note that function is about to be extended to be more !hugetlb like in Ryans work (smaller THP).
Quoted 4 lines
For
folio_add_file_rmap_range(): would it work if it just pass the hugetlb folio range into it? Would that make it much more difficult with the recent work on large folios from you or anyone?
Sure we can, that doesn't mean that it's a good interface for hugetlb that doesn't support ranges right now and maybe never will. :)
Quoted 8 lines
Quoted 5 lines
I don't think going in the other direction and e.g., removing
hugetlb_add_anon_rmap/hugetlb_add_new_anon_rmapis making a unification that is not really reasonable. It will only make things slower and the individual functions more complicated.IIUC so far the performance difference should be minimum on which helper to use.
I've learned that even the likely in "if (likely(compound))" matters in some rmap function.
So I'm still against effectively adding more folio_is_hugetlb() checks on hot code paths for the sake of "unification" that I don't really see as "unification".
Quoted 6 lines
As I mentioned, I sincerely don't know whether this patch is good or not as I don't know enough on everything around that is happening. It's just that I'll still think twice if to create hugetlb its own rmap API, because from the high level it's going the other way round to me. So I still want to raise this as a pure question.
I consider the end result here easier to read and easier to extend. If there are good reasons to go the different way, I'm all ears.
Signature · 3 lines
Cheers, David / dhildenb
PATCH 3/5 mm/rmap: introduce and use hugetlb_add_file_rmap()
In reply to David Hildenbrandhugetlb rmap handling differs quite a lot from "ordinary" rmap code, and we already have dedicated functions for adding anon hugetlb folios and removing hugetlb folios.
Right now we're using page_dup_file_rmap() in some cases where "ordinary" rmap code would have used page_add_file_rmap(). So let's introduce and use hugetlb_add_file_rmap() instead. We won't be adding a "hugetlb_dup_file_rmap()" functon for the fork() case, as it would be doing the same: "dup" is just an optimization for "add".
While this is a cleanup, this will also make it easier to change rmap handling for partially-mappable folios.
What remains is a single page_dup_file_rmap() call in fork() code.
--- include/linux/rmap.h | 7 +++++++ mm/hugetlb.c | 6 +++--- mm/migrate.c | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-)
include/linux/rmap.h +7 -0
@@ -208,6 +208,13 @@ void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,void hugetlb_add_new_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address); +static inline void hugetlb_add_file_rmap (struct folio *folio) +{ +VM_WARN_ON_FOLIO (folio_test_anon (folio), folio); + +atomic_inc (&folio->_entire_mapcount); +} +static inline void hugetlb_remove_rmap (struct folio *folio) {atomic_dec (&folio->_entire_mapcount);
mm/hugetlb.c +3 -3
@@ -5401,7 +5401,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,* sleep during the process. */ if (!folio_test_anon (pte_folio)) { -page_dup_file_rmap (&pte_folio->page, true); +hugetlb_add_file_rmap (pte_folio); }else if (page_try_dup_anon_rmap (&pte_folio->page, true, src_vma)) {pte_t src_pte_old = entry;
@@ -6272,7 +6272,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,if (anon_rmap)hugetlb_add_new_anon_rmap (folio, vma, haddr);else -page_dup_file_rmap (&folio->page, true); +hugetlb_add_file_rmap (folio); new_pte =make_huge_pte (vma, &folio->page, ((vma->vm_flags &VM_WRITE ) && (vma->vm_flags &VM_SHARED )));/*
@@ -6723,7 +6723,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, goto out_release_unlock;if (folio_in_pagecache) -page_dup_file_rmap (&folio->page, true); +hugetlb_add_file_rmap (folio);else hugetlb_add_new_anon_rmap (folio, dst_vma, dst_addr);
mm/migrate.c +1 -1
@@ -252,7 +252,7 @@ static bool remove_migration_pte(struct folio *folio,hugetlb_add_anon_rmap (folio, vma, pvmw.address, rmap_flags); else -page_dup_file_rmap (new ,true ); +hugetlb_add_file_rmap (folio );set_huge_pte_at (vma->vm_mm, pvmw.address, pvmw.pte, pte, psize); }else
Signature · 1 line
2.41.0
PATCH 4/5 mm/rmap: introduce and use hugetlb_try_dup_anon_rmap()
In reply to David Hildenbrandhugetlb rmap handling differs quite a lot from "ordinary" rmap code, and we already have dedicated functions for adding anon hugetlb folios and removing hugetlb folios.
So let's introduce and use hugetlb_try_dup_anon_rmap() to make all hugetlb handling use dedicated hugetlb_* rmap functions.
While this is a cleanup, this will also make it easier to change rmap handling for partially-mappable folios.
Note that is_device_private_page() does not apply to hugetlb.
--- include/linux/mm.h | 12 +++++++++--- include/linux/rmap.h | 15 +++++++++++++++ mm/hugetlb.c | 3 +-- 3 files changed, 25 insertions(+), 5 deletions(-)
include/linux/mm.h +9 -3
@@ -1953,15 +1953,21 @@ static inline bool page_maybe_dma_pinned(struct page *page)* * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. */ -static inline bool page_needs_cow_for_dma (struct vm_area_struct *vma, -struct page *page) +static inline bool folio_needs_cow_for_dma (struct vm_area_struct *vma, +struct folio *folio) {VM_BUG_ON (!(raw_read_seqcount (&vma->vm_mm->write_protect_seq) &1 ));if (!test_bit (MMF_HAS_PINNED , &vma->vm_mm->flags))return false; -return page_maybe_dma_pinned (page); +return folio_maybe_dma_pinned (folio); +} + +static inline bool page_needs_cow_for_dma (struct vm_area_struct *vma, +struct page *page) +{ +return folio_needs_cow_for_dma (vma,page_folio (page)); }/**
include/linux/rmap.h +15 -0
@@ -208,6 +208,21 @@ void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,void hugetlb_add_new_anon_rmap (struct folio *,struct vm_area_struct *,unsigned long address); +/* See page_try_dup_anon_rmap() */ +static inline int hugetlb_try_dup_anon_rmap (struct folio *folio, +struct vm_area_struct *vma) +{ +VM_WARN_ON_FOLIO (!folio_test_anon (folio), folio); + +if (PageAnonExclusive (&folio->page)) { +if (unlikely (folio_needs_cow_for_dma (vma, folio))) +return -EBUSY ; +ClearPageAnonExclusive (&folio->page); + } +atomic_inc (&folio->_entire_mapcount); +return 0 ; +} +static inline void hugetlb_add_file_rmap (struct folio *folio) {VM_WARN_ON_FOLIO (folio_test_anon (folio), folio);
mm/hugetlb.c +1 -2
@@ -5402,8 +5402,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,*/ if (!folio_test_anon (pte_folio)) {hugetlb_add_file_rmap (pte_folio); - }else if (page_try_dup_anon_rmap (&pte_folio->page, - true, src_vma)) { + }else if (hugetlb_try_dup_anon_rmap (pte_folio, src_vma)) {pte_t src_pte_old = entry;struct folio *new_folio;
Signature · 1 line
2.41.0
PATCH 5/5 mm/rmap: add hugetlb sanity checks
In reply to David HildenbrandLet's make sure we end up with the right folios in the right functions.
--- include/linux/rmap.h | 6 ++++++ mm/rmap.c | 6 ++++++ 2 files changed, 12 insertions(+)
include/linux/rmap.h +6 -0
@@ -212,6 +212,7 @@ void hugetlb_add_new_anon_rmap(struct folio *, struct vm_area_struct *,static inline int hugetlb_try_dup_anon_rmap (struct folio *folio,struct vm_area_struct *vma) { +VM_WARN_ON_FOLIO (!folio_test_hugetlb (folio), folio);VM_WARN_ON_FOLIO (!folio_test_anon (folio), folio);if (PageAnonExclusive (&folio->page)) {
@@ -225,6 +226,7 @@ static inline int hugetlb_try_dup_anon_rmap(struct folio *folio,static inline void hugetlb_add_file_rmap (struct folio *folio) { +VM_WARN_ON_FOLIO (!folio_test_hugetlb (folio), folio);VM_WARN_ON_FOLIO (folio_test_anon (folio), folio);atomic_inc (&folio->_entire_mapcount);
@@ -232,11 +234,15 @@ static inline void hugetlb_add_file_rmap(struct folio *folio)static inline void hugetlb_remove_rmap (struct folio *folio) { +VM_WARN_ON_FOLIO (!folio_test_hugetlb (folio), folio); +atomic_dec (&folio->_entire_mapcount); }static inline void __page_dup_rmap (struct page *page,bool compound) { +VM_WARN_ON (folio_test_hugetlb (page_folio (page))); +if (compound) {struct folio *folio = (struct folio *)page;
mm/rmap.c +6 -0
@@ -1313,6 +1313,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, {int nr; +VM_WARN_ON_FOLIO (folio_test_hugetlb (folio), folio);VM_BUG_ON_VMA (address < vma->vm_start || address >= vma->vm_end, vma);__folio_set_swapbacked (folio);
@@ -1353,6 +1354,7 @@ void folio_add_file_rmap_range(struct folio *folio, struct page *page,unsigned int nr_pmdmapped =0 , first;int nr =0 ; +VM_WARN_ON_FOLIO (folio_test_hugetlb (folio), folio);VM_WARN_ON_FOLIO (compound && !folio_test_pmd_mappable (folio), folio);/* Is page being mapped by PTE? Is this its first map to be added? */
@@ -1438,6 +1440,7 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma,bool last;enum node_stat_item idx; +VM_WARN_ON_FOLIO (folio_test_hugetlb (folio), folio);VM_BUG_ON_PAGE (compound && !PageHead (page), page);/* Is page being unmapped by PTE? Is this its last map to be removed? */
@@ -2585,6 +2588,7 @@ void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)void hugetlb_add_anon_rmap (struct folio *folio,struct vm_area_struct *vma,unsigned long address,rmap_t flags) { +VM_WARN_ON_FOLIO (!folio_test_hugetlb (folio), folio);VM_WARN_ON_FOLIO (!folio_test_anon (folio), folio);atomic_inc (&folio->_entire_mapcount);
@@ -2597,6 +2601,8 @@ void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,void hugetlb_add_new_anon_rmap (struct folio *folio,struct vm_area_struct *vma,unsigned long address) { +VM_WARN_ON_FOLIO (!folio_test_hugetlb (folio), folio); +BUG_ON (address < vma->vm_start || address >= vma->vm_end);/* increment count (starts at -1) */ atomic_set (&folio->_entire_mapcount,0 );
Signature · 1 line
2.41.0