mm: khugepaged: simplify scanning progress in pmd
Subsystems (2)
PATCH 1/1 mm: khugepaged: simplify scanning progress in pmd
From: Vernon Yang <[email protected]>
Placing "cur_progress" inside "struct collapse_control" makes the overall code simpler, there also coincidentally has a 4-bytes hole, as shown below:
struct collapse_control {
bool is_khugepaged; /* 0 1 */
/* XXX 3 bytes hole, try to pack */
u32 node_load[64]; /* 4 256 */
/* XXX 4 bytes hole, try to pack */
/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
nodemask_t alloc_nmask; /* 264 8 */
/* size: 272, cachelines: 5, members: 3 */
/* sum members: 265, holes: 2, sum holes: 7 */
/* last cacheline: 16 bytes */
};Also rename "cur_progress" to "progress_in_pmd", make it clearer.
No function changes.
--- mm/khugepaged.c | 51 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 26 deletions(-)
mm/khugepaged.c +25 -26
@@ -70,8 +70,8 @@ static struct task_struct *khugepaged_thread __read_mostly;static DEFINE_MUTEX (khugepaged_mutex);/* -* default scan 8*HPAGE_PMD_NR ptes, pmd_mapped, no_pte_table or vmas -* every 10 second. +* default scan 8*HPAGE_PMD_NR ptes, pte_mapped_hugepage, pmd_mapped, +* no_pte_table or vmas every 10 second. */ static unsigned int khugepaged_pages_to_scan__read_mostly ;static unsigned int khugepaged_pages_collapsed;
@@ -104,6 +104,12 @@ struct collapse_control {/* Num pages scanned per node */ u32 node_load[MAX_NUMNODES ]; +/* +* Num pages scanned per pmd, include ptes, +* pte_mapped_hugepage, pmd_mapped or no_pte_table. +*/ +unsigned int progress_in_pmd; +/* nodemask for allocation fallback */ nodemask_t alloc_nmask; };
@@ -1246,8 +1252,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long astatic enum scan_result hpage_collapse_scan_pmd (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long start_addr, -bool *mmap_locked,unsigned int *cur_progress, -struct collapse_control *cc) +bool *mmap_locked,struct collapse_control *cc) {pmd_t *pmd;pte_t *pte, *_pte;
@@ -1263,25 +1268,23 @@ static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm, result =find_pmd_or_thp_or_none (mm, start_addr, &pmd);if (result !=SCAN_SUCCEED ) { -if (cur_progress) - *cur_progress =1 ; + cc->progress_in_pmd =1 ; goto out; }memset (cc->node_load,0 ,sizeof (cc->node_load));nodes_clear (cc->alloc_nmask); + cc->progress_in_pmd =0 ; pte =pte_offset_map_lock (mm, pmd, start_addr, &ptl);if (!pte) { -if (cur_progress) - *cur_progress =1 ; + cc->progress_in_pmd =1 ; result =SCAN_NO_PTE_TABLE ; goto out; }for (addr = start_addr, _pte = pte; _pte < pte +HPAGE_PMD_NR ; _pte++, addr +=PAGE_SIZE ) { -if (cur_progress) - *cur_progress +=1 ; + cc->progress_in_pmd++;pte_t pteval =ptep_get (_pte);if (pte_none_or_zero (pteval)) {
@@ -2314,7 +2317,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,static enum scan_result hpage_collapse_scan_file (struct mm_struct *mm,unsigned long addr,struct file *file,pgoff_t start, -unsigned int *cur_progress,struct collapse_control *cc) +struct collapse_control *cc) {struct folio *folio =NULL ;struct address_space *mapping = file->f_mapping;
@@ -2327,6 +2330,7 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, swap =0 ;memset (cc->node_load,0 ,sizeof (cc->node_load));nodes_clear (cc->alloc_nmask); + cc->progress_in_pmd =0 ;rcu_read_lock ();xas_for_each (&xas, folio, start +HPAGE_PMD_NR -1 ) {if (xas_retry (&xas, folio))
@@ -2404,12 +2408,10 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, } }rcu_read_unlock (); -if (cur_progress) { -if (result ==SCAN_PTE_MAPPED_HUGEPAGE ) - *cur_progress =1 ; -else - *cur_progress =HPAGE_PMD_NR ; - } +if (result ==SCAN_PTE_MAPPED_HUGEPAGE ) + cc->progress_in_pmd =1 ; +else + cc->progress_in_pmd =HPAGE_PMD_NR ;if (result ==SCAN_SUCCEED ) { if (cc->is_khugepaged &&
@@ -2434,7 +2436,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultstruct mm_slot *slot;struct mm_struct *mm;struct vm_area_struct *vma; -int progress =0 ; +unsigned int progress =0 ;VM_BUG_ON (!pages);lockdep_assert_held (&khugepaged_mm_lock);
@@ -2488,7 +2490,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultwhile (khugepaged_scan.address < hend) {bool mmap_locked = true; -unsigned int cur_progress =0 ;cond_resched ();if (unlikely (hpage_collapse_test_exit_or_disable (mm)))
@@ -2505,8 +2506,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultmmap_read_unlock (mm); mmap_locked = false; *result =hpage_collapse_scan_file (mm, - khugepaged_scan.address, file, pgoff, - &cur_progress, cc); + khugepaged_scan.address, file, pgoff, cc);fput (file);if (*result ==SCAN_PTE_MAPPED_HUGEPAGE ) {mmap_read_lock (mm);
@@ -2520,8 +2520,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result } }else { *result =hpage_collapse_scan_pmd (mm, vma, - khugepaged_scan.address, &mmap_locked, - &cur_progress, cc); + khugepaged_scan.address, &mmap_locked, cc); }if (*result ==SCAN_SUCCEED )
@@ -2529,7 +2528,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result/* move to next address */ khugepaged_scan.address +=HPAGE_PMD_SIZE ; - progress += cur_progress; + progress += cc->progress_in_pmd;if (!mmap_locked)/* * We released mmap_lock so break loop. Note
@@ -2852,7 +2851,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, mmap_locked = false; *lock_dropped = true; result =hpage_collapse_scan_file (mm, addr, file, pgoff, -NULL , cc); + cc);if (result ==SCAN_PAGE_DIRTY_OR_WRITEBACK && !triggered_wb &&mapping_can_writeback (file->f_mapping)) {
@@ -2867,7 +2866,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,fput (file); }else { result =hpage_collapse_scan_pmd (mm, vma, addr, - &mmap_locked,NULL , cc); + &mmap_locked, cc); }if (!mmap_locked) *lock_dropped = true;
Signature · 1 line
2.51.0
David Hildenbrand (Arm)
maintainerQuoted 19 lines · Vernon Yang
From: Vernon Yang <[email protected]>
Placing "
cur_progress" inside "structcollapse_control" makes the overall code simpler, there also coincidentally has a 4-bytes hole, as shown below:struct collapse_control { bool is_khugepaged; /* 0 1 */ /* XXX 3 bytes hole, try to pack */ u32 node_load[64]; /* 4 256 */ /* XXX 4 bytes hole, try to pack */ /* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */ nodemask_t alloc_nmask; /* 264 8 */ /* size: 272, cachelines: 5, members: 3 */ /* sum members: 265, holes: 2, sum holes: 7 */ /* last cacheline: 16 bytes */ };
Quoted 6 lines
Also rename "
cur_progress" to "progress_in_pmd", make it clearer.No function changes.
Signed-off-by Vernon Yang <[email protected]>---
We should really squash that into the original patch and revisit it.
Because I think we might be able to do even better.
Could we just switch to "cc->progress" like so (modified original patch):
mm/khugepaged.c +31 -21
@@ -68,7 +68,10 @@ enum scan_result {static struct task_struct *khugepaged_thread__read_mostly ;static DEFINE_MUTEX (khugepaged_mutex); -/* default scan 8*HPAGE_PMD_NR ptes (or vmas) every 10 second */ +/* +* default scan 8*HPAGE_PMD_NR ptes, pte_mapped_hugepage, pmd_mapped, +* no_pte_table or vmas every 10 second. +*/ static unsigned int khugepaged_pages_to_scan__read_mostly ;static unsigned int khugepaged_pages_collapsed;static unsigned int khugepaged_full_scans;
@@ -100,6 +103,9 @@ struct collapse_control {/* Num pages scanned per node */ u32 node_load[MAX_NUMNODES ]; +/* Num pages scanned (see khugepaged_pages_to_scan). */ +unsigned int progress; +/* nodemask for allocation fallback */ nodemask_t alloc_nmask; };
@@ -1247,19 +1253,24 @@ static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm,VM_BUG_ON (start_addr & ~HPAGE_PMD_MASK ); result =find_pmd_or_thp_or_none (mm, start_addr, &pmd); -if (result !=SCAN_SUCCEED ) +if (result !=SCAN_SUCCEED ) { + cc->progress++; goto out; + }memset (cc->node_load,0 ,sizeof (cc->node_load));nodes_clear (cc->alloc_nmask); pte =pte_offset_map_lock (mm, pmd, start_addr, &ptl);if (!pte) { + cc->progress++; result =SCAN_NO_PTE_TABLE ; goto out; }for (addr = start_addr, _pte = pte; _pte < pte +HPAGE_PMD_NR ; _pte++, addr +=PAGE_SIZE ) { + cc->progress++; +pte_t pteval =ptep_get (_pte);if (pte_none_or_zero (pteval)) { ++none_or_zero;
@@ -2370,6 +2381,10 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, unsigned } }rcu_read_unlock (); +if (result ==SCAN_PTE_MAPPED_HUGEPAGE ) + cc->progress++; +else + cc->progress +=HPAGE_PMD_NR ;if (result ==SCAN_SUCCEED ) { if (cc->is_khugepaged &&
@@ -2385,8 +2400,8 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, unsignedreturn result; } -static unsigned int khugepaged_scan_mm_slot (unsigned int pages,enum scan_result *result, -struct collapse_control *cc) +static void khugepaged_scan_mm_slot (unsigned int progress_max, +enum scan_result *result,struct collapse_control *cc) __releases(&khugepaged_mm_lock) __acquires(&khugepaged_mm_lock) {
@@ -2394,9 +2409,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultstruct mm_slot *slot;struct mm_struct *mm;struct vm_area_struct *vma; -int progress =0 ; -VM_BUG_ON (!pages);lockdep_assert_held (&khugepaged_mm_lock); *result =SCAN_FAIL ;
@@ -2419,7 +2432,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultif (unlikely (!mmap_read_trylock (mm))) goto breakouterloop_mmap_lock; - progress++; + cc->progress++;if (unlikely (hpage_collapse_test_exit_or_disable (mm))) goto breakouterloop;
@@ -2429,17 +2442,17 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultcond_resched ();if (unlikely (hpage_collapse_test_exit_or_disable (mm))) { - progress++; + cc->progress++;break ; }if (!thp_vma_allowable_order (vma, vma->vm_flags,TVA_KHUGEPAGED ,PMD_ORDER )) { - progress++; + cc->progress++;continue ; } hstart =round_up (vma->vm_start,HPAGE_PMD_SIZE ); hend =round_down (vma->vm_end,HPAGE_PMD_SIZE );if (khugepaged_scan.address > hend) { - progress++; + cc->progress++;continue ; }if (khugepaged_scan.address < hstart)
@@ -2486,7 +2499,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result/* move to next address */ khugepaged_scan.address +=HPAGE_PMD_SIZE ; - progress +=HPAGE_PMD_NR ;if (!mmap_locked)/* * We released mmap_lock so break loop. Note
@@ -2496,7 +2508,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result* correct result back to caller. */ goto breakouterloop_mmap_lock; -if (progress >= pages) +if (cc->progress >= progress_max) goto breakouterloop; } }
@@ -2527,9 +2539,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultcollect_mm_slot (slot); } -trace_mm_khugepaged_scan (mm, progress, khugepaged_scan.mm_slot ==NULL ); - -return progress; +trace_mm_khugepaged_scan (mm, cc->progress, khugepaged_scan.mm_slot ==NULL ); }static int khugepaged_has_work (void )
@@ -2545,13 +2555,14 @@ static int khugepaged_wait_event(void)static void khugepaged_do_scan (struct collapse_control *cc) { -unsigned int progress =0 , pass_through_head =0 ; -unsigned int pages =READ_ONCE (khugepaged_pages_to_scan); +const unsigned int progress_max =READ_ONCE (khugepaged_pages_to_scan); +unsigned int pass_through_head =0 ;bool wait = true;enum scan_result result =SCAN_SUCCEED ;lru_add_drain_all (); + cc->progress =0 ;while (true) {cond_resched ();
@@ -2563,13 +2574,12 @@ static void khugepaged_do_scan(struct collapse_control *cc) pass_through_head++;if (khugepaged_has_work () && pass_through_head <2 ) - progress +=khugepaged_scan_mm_slot (pages - progress, - &result, cc); +khugepaged_scan_mm_slot (progress_max, &result, cc);else - progress = pages; + cc->progress = progress_max;spin_unlock (&khugepaged_mm_lock); -if (progress >= pages) +if (cc->progress >= progress_max)break ;if (result ==SCAN_ALLOC_HUGE_PAGE_FAIL ) {
Signature · 7 lines
2.43.0 -- Cheers, David
Quoted 20 lines · David Hildenbrand (Arm)
Quoted 19 lines · Vernon Yang
From: Vernon Yang <[email protected]>
Placing "
cur_progress" inside "structcollapse_control" makes the overall code simpler, there also coincidentally has a 4-bytes hole, as shown below:struct collapse_control { bool is_khugepaged; /* 0 1 */ /* XXX 3 bytes hole, try to pack */ u32 node_load[64]; /* 4 256 */ /* XXX 4 bytes hole, try to pack */ /* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */ nodemask_t alloc_nmask; /* 264 8 */ /* size: 272, cachelines: 5, members: 3 */ /* sum members: 265, holes: 2, sum holes: 7 */ /* last cacheline: 16 bytes */ };
Quoted 8 lines
Quoted 6 lines
Also rename "
cur_progress" to "progress_in_pmd", make it clearer.No function changes.
Signed-off-by Vernon Yang <[email protected]>---
We should really squash that into the original patch and revisit it.
Yes, squashing this patch into the original patch makes the git commit log clearer.
Quoted 151 lines
Because I think we might be able to do even better.
Could we just switch to "
cc->progress" like so (modified original patch):mm/khugepaged.c +25
-15@@ -68,7 +68,10 @@ enum scan_result {static struct task_struct *khugepaged_thread__read_mostly ;static DEFINE_MUTEX (khugepaged_mutex); -/* default scan 8*HPAGE_PMD_NR ptes (or vmas) every 10 second */ +/* +* default scan 8*HPAGE_PMD_NR ptes, pte_mapped_hugepage, pmd_mapped, +* no_pte_table or vmas every 10 second. +*/ static unsigned int khugepaged_pages_to_scan__read_mostly ;static unsigned int khugepaged_pages_collapsed;static unsigned int khugepaged_full_scans;@@ -100,6 +103,9 @@ struct collapse_control {/* Num pages scanned per node */ u32 node_load[MAX_NUMNODES ]; +/* Num pages scanned (see khugepaged_pages_to_scan). */ +unsigned int progress; +/* nodemask for allocation fallback */ nodemask_t alloc_nmask; };@@ -1247,19 +1253,24 @@ static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm,VM_BUG_ON (start_addr & ~HPAGE_PMD_MASK ); result =find_pmd_or_thp_or_none (mm, start_addr, &pmd); -if (result !=SCAN_SUCCEED ) +if (result !=SCAN_SUCCEED ) { + cc->progress++; goto out; + }memset (cc->node_load,0 ,sizeof (cc->node_load));nodes_clear (cc->alloc_nmask); pte =pte_offset_map_lock (mm, pmd, start_addr, &ptl);if (!pte) { + cc->progress++; result =SCAN_NO_PTE_TABLE ; goto out; }for (addr = start_addr, _pte = pte; _pte < pte +HPAGE_PMD_NR ; _pte++, addr +=PAGE_SIZE ) { + cc->progress++; +pte_t pteval =ptep_get (_pte);if (pte_none_or_zero (pteval)) { ++none_or_zero;@@ -2370,6 +2381,10 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, unsigned } }rcu_read_unlock (); +if (result ==SCAN_PTE_MAPPED_HUGEPAGE ) + cc->progress++; +else + cc->progress +=HPAGE_PMD_NR ;if (result ==SCAN_SUCCEED ) { if (cc->is_khugepaged &&@@ -2385,8 +2400,8 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm, unsignedreturn result; } -static unsigned int khugepaged_scan_mm_slot (unsigned int pages,enum scan_result *result, -struct collapse_control *cc) +static void khugepaged_scan_mm_slot (unsigned int progress_max, +enum scan_result *result,struct collapse_control *cc) __releases(&khugepaged_mm_lock) __acquires(&khugepaged_mm_lock) {@@ -2394,9 +2409,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultstruct mm_slot *slot;struct mm_struct *mm;struct vm_area_struct *vma; -int progress =0 ; -VM_BUG_ON (!pages);lockdep_assert_held (&khugepaged_mm_lock); *result =SCAN_FAIL ;@@ -2419,7 +2432,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultif (unlikely (!mmap_read_trylock (mm))) goto breakouterloop_mmap_lock; - progress++; + cc->progress++;if (unlikely (hpage_collapse_test_exit_or_disable (mm))) goto breakouterloop;@@ -2429,17 +2442,17 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultcond_resched ();if (unlikely (hpage_collapse_test_exit_or_disable (mm))) { - progress++; + cc->progress++;break ; }if (!thp_vma_allowable_order (vma, vma->vm_flags,TVA_KHUGEPAGED ,PMD_ORDER )) { - progress++; + cc->progress++;continue ; } hstart =round_up (vma->vm_start,HPAGE_PMD_SIZE ); hend =round_down (vma->vm_end,HPAGE_PMD_SIZE );if (khugepaged_scan.address > hend) { - progress++; + cc->progress++;continue ; }if (khugepaged_scan.address < hstart)@@ -2486,7 +2499,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result/* move to next address */ khugepaged_scan.address +=HPAGE_PMD_SIZE ; - progress +=HPAGE_PMD_NR ;if (!mmap_locked)/* * We released mmap_lock so break loop. Note @@ -2496,7 +2508,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result* correct result back to caller. */ goto breakouterloop_mmap_lock; -if (progress >= pages) +if (cc->progress >= progress_max) goto breakouterloop; } }@@ -2527,9 +2539,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_resultcollect_mm_slot (slot); } -trace_mm_khugepaged_scan (mm, progress, khugepaged_scan.mm_slot ==NULL ); - -return progress; +trace_mm_khugepaged_scan (mm, cc->progress, khugepaged_scan.mm_slot ==NULL ); }static int khugepaged_has_work (void )@@ -2545,13 +2555,14 @@ static int khugepaged_wait_event(void)static void khugepaged_do_scan (struct collapse_control *cc) {
mm/khugepaged.c static int khugepaged_wait_event(void) · line 2565
Quoted 10 lines
mm/khugepaged.c +3
-2-unsigned int progress =0 , pass_through_head =0 ; -unsigned int pages =READ_ONCE (khugepaged_pages_to_scan); +const unsigned int progress_max =READ_ONCE (khugepaged_pages_to_scan); +unsigned int pass_through_head =0 ;bool wait = true;enum scan_result result =SCAN_SUCCEED ;lru_add_drain_all (); + cc->progress =0 ;
madvise(MADV_COLLAPSE) is missing the initialization of "cc->progress" because "cc = kmalloc_obj(*cc)". We need to manually initialize it in madvise_collapse().
LGTM, I will submit v2 with this patch and squashed into the original patch.
--
Cheers,
Vernon
David Hildenbrand (Arm)
maintainerQuoted 13 lines
Quoted 13 lines
static int
khugepaged_has_work(void)mm/khugepaged.c +2
-2@@ -2545,13 +2555,14 @@ static int khugepaged_wait_event(void)static void khugepaged_do_scan (struct collapse_control *cc) { -unsigned int progress =0 , pass_through_head =0 ; -unsigned int pages =READ_ONCE (khugepaged_pages_to_scan); +const unsigned int progress_max =READ_ONCE (khugepaged_pages_to_scan); +unsigned int pass_through_head =0 ;bool wait = true;enum scan_result result =SCAN_SUCCEED ;
Quoted 7 lines
mm/khugepaged.c static int khugepaged_wait_event(void) · line 2565
Quoted 3 lines
mm/khugepaged.c +1
-0lru_add_drain_all (); + cc->progress =0 ;
madvise(MADV_COLLAPSE)is missing the initialization of "cc->progress" because "cc =kmalloc_obj(*cc)". We need to manually initialize it inmadvise_collapse().
Right, although it wouldn't matter right now, because nobody would be reading the value.
It's really weird that cc is not simply completely zeroed out in madvise_collapse().
Signature · 3 lines
Cheers, David