support shmem mTHP collapse
Subsystems (4)
Baolin Wang
reviewerPATCH 1/5 mm: khugepaged: expand the is_refcount_suitable() to support file folios
In reply to Baolin WangExpand the is_refcount_suitable() to support reference checks for file folios, as preparation for supporting shmem mTHP collapse.
--- mm/khugepaged.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
mm/khugepaged.c +8 -3
@@ -549,8 +549,14 @@ static bool is_refcount_suitable(struct folio *folio)int expected_refcount; expected_refcount =folio_mapcount (folio); -if (folio_test_swapcache (folio)) +if (folio_test_anon (folio)) { + expected_refcount +=folio_test_swapcache (folio) ? +folio_nr_pages (folio) :0 ; + }else { expected_refcount +=folio_nr_pages (folio); +if (folio_test_private (folio)) + expected_refcount++; + }return folio_ref_count (folio) == expected_refcount; }
@@ -2285,8 +2291,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,break ; } -if (folio_ref_count (folio) != -1 +folio_mapcount (folio) +folio_test_private (folio)) { +if (!is_refcount_suitable (folio)) { result =SCAN_PAGE_COUNT ;break ; }
Signature · 1 line
2.39.3
Baolin Wang
reviewerPATCH 2/5 mm: khugepaged: use the number of pages in the folio to check the reference count
In reply to Baolin WangUse the number of pages in the folio to check the reference count as preparation for supporting shmem mTHP collapse.
--- mm/khugepaged.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
mm/khugepaged.c +3 -3
@@ -1994,7 +1994,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,/* * We control three references to the folio: * - we hold a pin on it; -* - one reference from page cache; +* - nr_pages reference from page cache; * - one from lru_isolate_folio; * If those are the only references, then any new usage * of the folio will have to fetch it from the page
@@ -2002,7 +2002,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,* truncate, so any new usage will be blocked until we * unlock folio after collapse/during rollback. */ -if (folio_ref_count (folio) !=3 ) { +if (folio_ref_count (folio) !=2 +folio_nr_pages (folio)) { result =SCAN_PAGE_COUNT ;xas_unlock_irq (&xas);folio_putback_lru (folio);
@@ -2185,7 +2185,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,folio_clear_active (folio);folio_clear_unevictable (folio);folio_unlock (folio); -folio_put_refs (folio,3 ); +folio_put_refs (folio,2 +folio_nr_pages (folio)); } goto out;
Signature · 1 line
2.39.3
Baolin Wang
reviewercover support shmem mTHP collapse
Hi,
Shmem already supports mTHP allocation[1], and this patch set adds support for shmem mTHP collapse, as well as adding relevant test cases. Please help to review. Thanks.
Note: all khugepaged selftests have passed.
[1] https://lore.kernel.org/all/[email protected]/T/#m4bd7e701c7b5f36f712055e4360cad593a22b3bf
Baolin Wang (5):
mm: khugepaged: expand the is_refcount_suitable() to support file
folios
mm: khugepaged: use the number of pages in the folio to check the
reference count
mm: khugepaged: support shmem mTHP copy
mm: khugepaged: support shmem mTHP collapse
selftests: mm: support shmem mTHP collapse testing
mm/khugepaged.c | 60 ++++++++++++----------- tools/testing/selftests/mm/khugepaged.c | 4 +- tools/testing/selftests/mm/thp_settings.c | 46 ++++++++++++++--- tools/testing/selftests/mm/thp_settings.h | 9 +++- 4 files changed, 83 insertions(+), 36 deletions(-)
Signature · 1 line
2.39.3
Baolin Wang
reviewerPATCH 3/5 mm: khugepaged: support shmem mTHP copy
Iterate each subpage in the large folio to copy, as preparation for supporting shmem mTHP collapse.
--- mm/khugepaged.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
mm/khugepaged.c +10 -5
@@ -2060,17 +2060,22 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, index = start; dst =folio_page (new_folio,0 );list_for_each_entry (folio, &pagelist, lru) { +int i, nr_pages =folio_nr_pages (folio); +while (index < folio->index) {clear_highpage (dst); index++; dst++; } -if (copy_mc_highpage (dst,folio_page (folio,0 )) >0 ) { - result =SCAN_COPY_MC ; - goto rollback; + +for (i =0 ; i < nr_pages; i++) { +if (copy_mc_highpage (dst,folio_page (folio, i)) >0 ) { + result =SCAN_COPY_MC ; + goto rollback; + } + index++; + dst++; } - index++; - dst++; }while (index < end) {clear_highpage (dst);
Signature · 1 line
2.39.3
Baolin Wang
reviewerPATCH 4/5 mm: khugepaged: support shmem mTHP collapse
In reply to Baolin WangShmem already supports the allocation of mTHP, but khugepaged does not yet support collapsing mTHP folios. Now khugepaged is ready to support mTHP, and this patch enables the collapse of shmem mTHP.
--- mm/khugepaged.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-)
mm/khugepaged.c +11 -17
@@ -1847,7 +1847,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } }while (1 ); -for (index = start; index < end; index++) { +for (index = start; index < end;) {xas_set (&xas, index); folio =xas_load (&xas);
@@ -1866,6 +1866,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } } nr_none++; + index++;continue ; }
@@ -1947,12 +1948,10 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,* we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ -if (folio_test_large (folio)) { - result =folio_order (folio) ==HPAGE_PMD_ORDER && - folio->index == start -/* Maybe PMD-mapped */ - ?SCAN_PTE_MAPPED_HUGEPAGE - :SCAN_PAGE_COMPOUND ; +if (folio_order (folio) ==HPAGE_PMD_ORDER && + folio->index == start) { +/* Maybe PMD-mapped */ + result =SCAN_PTE_MAPPED_HUGEPAGE ; goto out_unlock; }
@@ -2013,6 +2012,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,* Accumulate the folios that are being collapsed. */ list_add_tail (&folio->lru, &pagelist); + index +=folio_nr_pages (folio);continue ; out_unlock:folio_unlock (folio);
@@ -2265,16 +2265,10 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,continue ; } -/* -* TODO: khugepaged should compact smaller compound pages -* into a PMD sized page -*/ -if (folio_test_large (folio)) { - result =folio_order (folio) ==HPAGE_PMD_ORDER && - folio->index == start -/* Maybe PMD-mapped */ - ?SCAN_PTE_MAPPED_HUGEPAGE - :SCAN_PAGE_COMPOUND ; +if (folio_order (folio) ==HPAGE_PMD_ORDER && + folio->index == start) { +/* Maybe PMD-mapped */ + result =SCAN_PTE_MAPPED_HUGEPAGE ;/* * For SCAN_PTE_MAPPED_HUGEPAGE, further processing * by the caller won't touch the page cache, and so
Signature · 1 line
2.39.3
Baolin Wang
reviewerPATCH 5/5 selftests: mm: support shmem mTHP collapse testing
In reply to Baolin WangAdd shmem mTHP collpase testing. Similar to the anonymous page, users can use the '-s' parameter to specify the shmem mTHP size for testing.
--- tools/testing/selftests/mm/khugepaged.c | 4 +- tools/testing/selftests/mm/thp_settings.c | 46 ++++++++++++++++++++--- tools/testing/selftests/mm/thp_settings.h | 9 ++++- 3 files changed, 51 insertions(+), 8 deletions(-)
tools/testing/selftests/mm/khugepaged.c +3 -1
@@ -1095,7 +1095,7 @@ static void usage(void)fprintf (stderr,"\n\tSupported Options:\n" );fprintf (stderr,"\t\t-h: This help message.\n" );fprintf (stderr,"\t\t-s: mTHP size, expressed as page order.\n" ); -fprintf (stderr,"\t\t Defaults to 0. Use this size for anon allocations.\n" ); +fprintf (stderr,"\t\t Defaults to 0. Use this size for anon or shmem allocations.\n" );exit (1 ); }
@@ -1209,6 +1209,8 @@ int main(int argc, char **argv) default_settings.khugepaged.pages_to_scan = hpage_pmd_nr *8 ; default_settings.hugepages[hpage_pmd_order].enabled =THP_INHERIT ; default_settings.hugepages[anon_order].enabled =THP_ALWAYS ; + default_settings.shmem_hugepages[hpage_pmd_order].enabled =SHMEM_INHERIT ; + default_settings.shmem_hugepages[anon_order].enabled =SHMEM_ALWAYS ;save_settings ();thp_push_settings (&default_settings);
tools/testing/selftests/mm/thp_settings.c +40 -6
94 lines not on this page · raw message
tools/testing/selftests/mm/thp_settings.h +8 -1
@@ -22,10 +22,11 @@ enum thp_defrag { };enum shmem_enabled { +SHMEM_NEVER ,SHMEM_ALWAYS ,SHMEM_WITHIN_SIZE ,SHMEM_ADVISE , -SHMEM_NEVER , +SHMEM_INHERIT ,SHMEM_DENY ,SHMEM_FORCE , };
@@ -46,6 +47,10 @@ struct khugepaged_settings {unsigned long pages_to_scan; }; +struct shmem_hugepages_settings { +enum shmem_enabled enabled; +}; +struct thp_settings {enum thp_enabled thp_enabled;enum thp_defrag thp_defrag;
@@ -54,6 +59,7 @@ struct thp_settings {struct khugepaged_settings khugepaged;unsigned long read_ahead_kb;struct hugepages_settings hugepages[NR_ORDERS ]; +struct shmem_hugepages_settings shmem_hugepages[NR_ORDERS ]; };int read_file (const char *path,char *buf,size_t buflen);
@@ -76,5 +82,6 @@ void thp_save_settings(void);void thp_set_read_ahead_path (char *path);unsigned long thp_supported_orders (void ); +unsigned long thp_shmem_supported_orders (void );#endif /* __THP_SETTINGS_H__ */
Signature · 1 line
2.39.3
Re: [PATCH 1/5] mm: khugepaged: expand the is_refcount_suitable() to support file folios
In reply to Baolin WangQuoted 15 lines · Baolin Wang
Expand the
is_refcount_suitable()to support reference checks for file folios, as preparation for supporting shmem mTHP collapse.Signed-off-by Baolin Wang <[email protected]>--- mm/khugepaged.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)mm/khugepaged.c +0
-0@@ -549,8 +549,14 @@ static bool is_refcount_suitable(struct folio *folio)int expected_refcount;
mm/khugepaged.c static bool is_refcount_suitable(struct folio *folio) · line 559
Quoted 10 lines
mm/khugepaged.c +7
-1expected_refcount =folio_mapcount (folio); -if (folio_test_swapcache (folio)) +if (folio_test_anon (folio)) { + expected_refcount +=folio_test_swapcache (folio) ? +folio_nr_pages (folio) :0 ; + }else { expected_refcount +=folio_nr_pages (folio); +if (folio_test_private (folio)) + expected_refcount++; + }
Alternatively, a bit neater
if (!folio_test_anon(folio) || folio_test_swapcache(folio))
expected_refcount += folio_nr_pages(folio); if (folio_test_private(folio)) expected_refcount++;
The latter check should be fine even for anon folios (although always false)
mm/khugepaged.c static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, · line 2289
Quoted 9 lines
mm/khugepaged.c +0
-2return folio_ref_count (folio) == expected_refcount; }@@ -2285,8 +2291,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,break ; } -if (folio_ref_count (folio) != -1 +folio_mapcount (folio) +folio_test_private (folio)) {
The "1" is due to the pagecache, right? IIUC, we don't hold a raised folio refcount as we do the xas_for_each().
Signature · 3 lines
Cheers, David / dhildenb
Baolin Wang
reviewerRe: [PATCH 1/5] mm: khugepaged: expand the is_refcount_suitable() to support file folios
Quoted 29 lines · David Hildenbrand
Quoted 15 lines · Baolin Wang
Expand the
is_refcount_suitable()to support reference checks for file folios, as preparation for supporting shmem mTHP collapse.Signed-off-by Baolin Wang <[email protected]>---
mm/khugepaged.c| 11 ++++++++---1 file changed, 8 insertions(+), 3 deletions(-)mm/khugepaged.c +0
-0@@ -549,8 +549,14 @@ static bool is_refcount_suitable(struct folio *folio) int expected_refcount;mm/khugepaged.c static bool is_refcount_suitable(struct folio · line 559
Quoted 10 lines
mm/khugepaged.c +7
-1expected_refcount = folio_mapcount(folio); - if (folio_test_swapcache(folio)) + if (folio_test_anon(folio)) { + expected_refcount += folio_test_swapcache(folio) ? + folio_nr_pages(folio) : 0; + } else { expected_refcount += folio_nr_pages(folio); + if (folio_test_private(folio)) + expected_refcount++; + }Alternatively, a bit neater
Quoted 6 lines
if (!folio_test_anon(folio) ||
folio_test_swapcache(folio))
expected_refcount +=folio_nr_pages(folio);
if (folio_test_private(folio))
expected_refcount++;The latter check should be fine even for anon folios (although always false)
Looks better. Will do in v2.
Quoted 10 lines
mm/khugepaged.c static int hpage_collapse_scan_file(struct · line 2289
Quoted 8 lines
mm/khugepaged.c +0
-2return folio_ref_count(folio) == expected_refcount; }@@ -2285,8 +2291,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, break; } - if (folio_ref_count(folio) != - 1 + folio_mapcount(folio) + folio_test_private(folio)) {The "1" is due to the pagecache, right? IIUC, we don't hold a raised folio refcount as we do the
xas_for_each().
Right.
Re: [PATCH 2/5] mm: khugepaged: use the number of pages in the folio to check the reference count
In reply to Baolin WangQuoted 13 lines · Baolin Wang
Use the number of pages in the folio to check the reference count as preparation for supporting shmem mTHP collapse.
Signed-off-by Baolin Wang <[email protected]>--- mm/khugepaged.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)mm/khugepaged.c +0
-0
mm/khugepaged.c static int collapse_file(struct mm_struct *mm, unsigned long addr, · line 1995
Quoted 2 lines
mm/khugepaged.c +0
-0@@ -1994,7 +1994,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,/* * We control three references to the folio:
^ "three" is wrong now.
Quoted 16 lines
mm/khugepaged.c +2
-2* - we hold a pin on it; -* - one reference from page cache; +* - nr_pages reference from page cache; * - one from lru_isolate_folio; * If those are the only references, then any new usage * of the folio will have to fetch it from the page @@ -2002,7 +2002,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,* truncate, so any new usage will be blocked until we * unlock folio after collapse/during rollback. */ -if (folio_ref_count (folio) !=3 ) { +if (folio_ref_count (folio) !=2 +folio_nr_pages (folio)) { result =SCAN_PAGE_COUNT ;xas_unlock_irq (&xas);folio_putback_lru (folio);
mm/khugepaged.c static int collapse_file(struct mm_struct *mm, unsigned long addr, · line 2191
Quoted 8 lines
mm/khugepaged.c +1
-1@@ -2185,7 +2185,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,folio_clear_active (folio);folio_clear_unevictable (folio);folio_unlock (folio); -folio_put_refs (folio,3 ); +folio_put_refs (folio,2 +folio_nr_pages (folio)); } goto out;
Signature · 3 lines
Cheers, David / dhildenb
Baolin Wang
reviewerRe: [PATCH 2/5] mm: khugepaged: use the number of pages in the folio to check the reference count
Quoted 9 lines · David Hildenbrand
Quoted 8 lines · Baolin Wang
Use the number of pages in the folio to check the reference count as preparation for supporting shmem mTHP collapse.
Signed-off-by Baolin Wang <[email protected]>---
mm/khugepaged.c| 6 +++---1 file changed, 3 insertions(+), 3 deletions(-)
Quoted 10 lines
mm/khugepaged.c static int collapse_file(struct mm_struct *mm, · line 1996
Quoted 8 lines
mm/khugepaged.c +0
-0@@ -1994,7 +1994,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, /** We control three references to the folio: ^ "three" is wrong now.
Ah, good catch. Will change to '2 + nr_pages'.
Quoted 20 lines
Quoted 20 lines
mm/khugepaged.c +1
-1* - we hold a pin on it; - * - one reference from page cache; + * - nr_pages reference from page cache; * - one from lru_isolate_folio; * If those are the only references, then any new usage* of the folio will have to fetch it from the page
mm/khugepaged.c +1
-1@@ -2002,7 +2002,7 @@ static int collapse_file(struct mm_struct *mm,unsigned long addr, * truncate, so any new usage will be blocked until we * unlock folio after collapse/during rollback. */ - if (folio_ref_count(folio) != 3) { + if (folio_ref_count(folio) != 2 + folio_nr_pages(folio)) { result = SCAN_PAGE_COUNT; xas_unlock_irq(&xas);
folio_putback_lru(folio);mm/khugepaged.c +0
-0@@ -2185,7 +2185,7 @@ static int collapse_file(struct mm_struct *mm,unsigned long addr, folio_clear_active(folio);
Quoted 8 lines
mm/khugepaged.c static int collapse_file(struct mm_struct *mm, unsigned long addr, · line 2191
Quoted 6 lines
mm/khugepaged.c +1
-1folio_clear_unevictable(folio); folio_unlock(folio); - folio_put_refs(folio, 3); + folio_put_refs(folio, 2 + folio_nr_pages(folio)); } goto out;
Thanks for reviewing.