mm/ksm: fix some accounting problems
Subsystems (4)
Chengming Zhou
reviewerv2 cover mm/ksm: fix some accounting problems
Changes in v2:
- Add Fixes and Acked-by tags from David Hildenbrand, thanks!
- Inline atomic_long_read(&ksm_zero_pages), per David Hildenbrand.
- Don't include the last two optimization patches to keep this fix
simple.
- Link to v1: https://lore.kernel.org/r/[email protected]
We encounter some abnormal ksm_pages_scanned and ksm_zero_pages during some random tests.
1. ksm_pages_scanned unchanged even ksmd scanning has progress.
2. ksm_zero_pages maybe -1 in some rare cases.
Thanks for review and comments!
---
Chengming Zhou (2):
mm/ksm: fix ksm_pages_scanned accounting
mm/ksm: fix ksm_zero_pages accounting
fs/proc/base.c | 2 +- include/linux/ksm.h | 17 ++++++++++++++--- include/linux/mm_types.h | 2 +- mm/ksm.c | 17 +++++++---------- 4 files changed, 23 insertions(+), 15 deletions(-)
---
base-commit: fb0f40125feec3de7ef4524600ac83946207117e
change-id: 20240508-b4-ksm-counters-04817b40d3ee
Best regards,
Signature · 1 line
Chengming Zhou <[email protected]>
Chengming Zhou
reviewerPATCH v2 1/2 mm/ksm: fix ksm_pages_scanned accounting
During testing, I found ksm_pages_scanned is unchanged although the scan_get_next_rmap_item() did return valid rmap_item that is not NULL.
The reason is the scan_get_next_rmap_item() will return NULL after a full scan, so ksm_do_scan() just return without accounting of the ksm_pages_scanned.
Fix it by just putting ksm_pages_scanned accounting in that loop, and it will be accounted more timely if that loop would last for a long time.
--- mm/ksm.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
mm/ksm.c +2 -4
@@ -2753,18 +2753,16 @@ static void ksm_do_scan(unsigned int scan_npages) {struct ksm_rmap_item *rmap_item;struct page *page; -unsigned int npages = scan_npages; -while (npages-- &&likely (!freezing (current))) { +while (scan_npages-- &&likely (!freezing (current))) {cond_resched (); rmap_item =scan_get_next_rmap_item (&page);if (!rmap_item)return ;cmp_and_merge_page (page, rmap_item);put_page (page); + ksm_pages_scanned++; } - - ksm_pages_scanned += scan_npages - npages; }static int ksmd_should_run (void )
Signature · 1 line
2.45.0
PATCH v2 1/2 mm/ksm: fix ksm_pages_scanned accounting
Quoted 22 lines
Fixes b348b5fe2b5f ("mm/ksm: add pages scanned metric")Signed-off-by Chengming Zhou <[email protected]>---
mm/ksm.c| 6 ++----1 file changed, 2 insertions(+), 4 deletions(-)mm/ksm.c +1
-2@@ -2753,18 +2753,16 @@ static void ksm_do_scan(unsigned int scan_npages) {struct ksm_rmap_item *rmap_item;struct page *page; -unsigned int npages = scan_npages; -while (npages-- &&likely (!freezing (current))) { +while (scan_npages-- &&likely (!freezing (current))) {cond_resched (); rmap_item =scan_get_next_rmap_item (&page);if (!rmap_item)
mm/ksm.c static void ksm_do_scan(unsigned int scan_npages) · line 2768
Quoted 10 lines
mm/ksm.c +1
-2return ;cmp_and_merge_page (page, rmap_item);put_page (page); + ksm_pages_scanned++; } - - ksm_pages_scanned += scan_npages - npages; }static int ksmd_should_run (void )
Looks good to me.
Chengming Zhou
reviewerPATCH v2 2/2 mm/ksm: fix ksm_zero_pages accounting
In reply to Chengming ZhouWe normally ksm_zero_pages++ in ksmd when page is merged with zero page, but ksm_zero_pages-- is done from page tables side, which can't protected by the ksmd mutex.
So we can read very exceptional value of ksm_zero_pages in rare cases, such as -1, which is very confusing to users.
Fix it by changing to use atomic_long_t, and the same case with the mm->ksm_zero_pages.
--- fs/proc/base.c | 2 +- include/linux/ksm.h | 17 ++++++++++++++--- include/linux/mm_types.h | 2 +- mm/ksm.c | 11 +++++------ 4 files changed, 21 insertions(+), 11 deletions(-)
fs/proc/base.c +1 -1
@@ -3214,7 +3214,7 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm =get_task_mm (task);if (mm) {seq_printf (m,"ksm_rmap_items %lu\n" , mm->ksm_rmap_items); -seq_printf (m,"ksm_zero_pages %lu\n" , mm->ksm_zero_pages); +seq_printf (m,"ksm_zero_pages %ld\n" ,mm_ksm_zero_pages (mm));seq_printf (m,"ksm_merging_pages %lu\n" , mm->ksm_merging_pages);seq_printf (m,"ksm_process_profit %ld\n" ,ksm_process_profit (mm));mmput (mm);
include/linux/ksm.h +14 -3
@@ -33,16 +33,27 @@ void __ksm_exit(struct mm_struct *mm);*/ #define is_ksm_zero_pte (pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) -extern unsigned long ksm_zero_pages; +extern atomic_long_t ksm_zero_pages; + +static inline void ksm_map_zero_page (struct mm_struct *mm) +{ +atomic_long_inc (&ksm_zero_pages); +atomic_long_inc (&mm->ksm_zero_pages); +}static inline void ksm_might_unmap_zero_page (struct mm_struct *mm,pte_t pte) {if (is_ksm_zero_pte (pte)) { - ksm_zero_pages--; - mm->ksm_zero_pages--; +atomic_long_dec (&ksm_zero_pages); +atomic_long_dec (&mm->ksm_zero_pages); } } +static inline long mm_ksm_zero_pages (struct mm_struct *mm) +{ +return atomic_long_read (&mm->ksm_zero_pages); +} +static inline int ksm_fork (struct mm_struct *mm,struct mm_struct *oldmm) {if (test_bit (MMF_VM_MERGEABLE , &oldmm->flags))
include/linux/mm_types.h +1 -1
@@ -985,7 +985,7 @@ struct mm_struct {* Represent how many empty pages are merged with kernel zero * pages when enabling KSM use_zero_pages. */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages;#endif /* CONFIG_KSM */ #ifdef CONFIG_LRU_GEN_WALKS_MMU struct {
mm/ksm.c +5 -6
@@ -296,7 +296,7 @@ static bool ksm_use_zero_pages __read_mostly;static bool ksm_smart_scan = true;/* The number of zero pages which is placed by KSM */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages =ATOMIC_LONG_INIT (0 );/* The number of pages that have been skipped due to "smart scanning" */ static unsigned long ksm_pages_skipped;
@@ -1429,8 +1429,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,* the dirty bit in zero page's PTE is set. */ newpte =pte_mkdirty (pte_mkspecial (pfn_pte (page_to_pfn (kpage), vma->vm_page_prot))); - ksm_zero_pages++; - mm->ksm_zero_pages++; +ksm_map_zero_page (mm);/* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we
@@ -3373,7 +3372,7 @@ static void wait_while_offlining(void)#ifdef CONFIG_PROC_FS long ksm_process_profit (struct mm_struct *mm) { -return (long )(mm->ksm_merging_pages + mm->ksm_zero_pages) *PAGE_SIZE - +return (long )(mm->ksm_merging_pages +mm_ksm_zero_pages (mm)) *PAGE_SIZE - mm->ksm_rmap_items *sizeof (struct ksm_rmap_item ); }#endif /* CONFIG_PROC_FS */
@@ -3662,7 +3661,7 @@ KSM_ATTR_RO(pages_skipped);static ssize_t ksm_zero_pages_show (struct kobject *kobj,struct kobj_attribute *attr,char *buf) { -return sysfs_emit (buf,"%ld\n" , ksm_zero_pages); +return sysfs_emit (buf,"%ld\n" ,atomic_long_read (&ksm_zero_pages)); }KSM_ATTR_RO (ksm_zero_pages);
@@ -3671,7 +3670,7 @@ static ssize_t general_profit_show(struct kobject *kobj, {long general_profit; - general_profit = (ksm_pages_sharing + ksm_zero_pages) *PAGE_SIZE - + general_profit = (ksm_pages_sharing +atomic_long_read (&ksm_zero_pages)) *PAGE_SIZE - ksm_rmap_items *sizeof (struct ksm_rmap_item );return sysfs_emit (buf,"%ld\n" , general_profit);
Signature · 1 line
2.45.0
Re: [PATCH v2 2/2] mm/ksm: fix ksm_zero_pages accounting
Quoted 3 lines
We normally ksm_zero_pages++ in ksmd when page is merged with zero page, but ksm_zero_pages-- is done from page tables side, which can't protected by the ksmd mutex.
"cant protected" -> "can't be protected".
But It's better to say "where there is no any accessing protection of ksm_zero_pages" because ksmd mutex is to protect the flag of ksm_run, not to protect the counters of KSM.
Anyway, The following code looks OK to me.
Quoted 133 lines
So we can read very exceptional value of
ksm_zero_pagesin rare cases, such as -1, which is very confusing to users.Fix it by changing to use
atomic_long_t, and the same case with themm->ksm_zero_pages.Fixes e2942062e01d ("ksm: count all zero pages placed by KSM")Fixes 6080d19f0704 ("ksm: add ksm zero pages for each process")Signed-off-by Chengming Zhou <[email protected]>--- fs/proc/base.c | 2 +- include/linux/ksm.h | 17 ++++++++++++++--- include/linux/mm_types.h | 2 +- mm/ksm.c | 11 +++++------ 4 files changed, 21 insertions(+), 11 deletions(-)fs/proc/base.c +1
-1@@ -3214,7 +3214,7 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm =get_task_mm (task);if (mm) {seq_printf (m,"ksm_rmap_items %lu\n" , mm->ksm_rmap_items); -seq_printf (m,"ksm_zero_pages %lu\n" , mm->ksm_zero_pages); +seq_printf (m,"ksm_zero_pages %ld\n" ,mm_ksm_zero_pages (mm));seq_printf (m,"ksm_merging_pages %lu\n" , mm->ksm_merging_pages);seq_printf (m,"ksm_process_profit %ld\n" ,ksm_process_profit (mm));mmput (mm);include/linux/ksm.h +14
-3@@ -33,16 +33,27 @@ void __ksm_exit(struct mm_struct *mm);*/ #define is_ksm_zero_pte (pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) -extern unsigned long ksm_zero_pages; +extern atomic_long_t ksm_zero_pages; + +static inline void ksm_map_zero_page (struct mm_struct *mm) +{ +atomic_long_inc (&ksm_zero_pages); +atomic_long_inc (&mm->ksm_zero_pages); +}static inline void ksm_might_unmap_zero_page (struct mm_struct *mm,pte_t pte) {if (is_ksm_zero_pte (pte)) { - ksm_zero_pages--; - mm->ksm_zero_pages--; +atomic_long_dec (&ksm_zero_pages); +atomic_long_dec (&mm->ksm_zero_pages); } } +static inline long mm_ksm_zero_pages (struct mm_struct *mm) +{ +return atomic_long_read (&mm->ksm_zero_pages); +} +static inline int ksm_fork (struct mm_struct *mm,struct mm_struct *oldmm) {if (test_bit (MMF_VM_MERGEABLE , &oldmm->flags))include/linux/mm_types.h +1
-1@@ -985,7 +985,7 @@ struct mm_struct {* Represent how many empty pages are merged with kernel zero * pages when enabling KSM use_zero_pages. */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages;#endif /* CONFIG_KSM */ #ifdef CONFIG_LRU_GEN_WALKS_MMU struct {mm/ksm.c +5
-6@@ -296,7 +296,7 @@ static bool ksm_use_zero_pages __read_mostly;static bool ksm_smart_scan = true;/* The number of zero pages which is placed by KSM */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages =ATOMIC_LONG_INIT (0 );/* The number of pages that have been skipped due to "smart scanning" */ static unsigned long ksm_pages_skipped;@@ -1429,8 +1429,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,* the dirty bit in zero page's PTE is set. */ newpte =pte_mkdirty (pte_mkspecial (pfn_pte (page_to_pfn (kpage), vma->vm_page_prot))); - ksm_zero_pages++; - mm->ksm_zero_pages++; +ksm_map_zero_page (mm);/* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we @@ -3373,7 +3372,7 @@ static void wait_while_offlining(void)#ifdef CONFIG_PROC_FS long ksm_process_profit (struct mm_struct *mm) { -return (long )(mm->ksm_merging_pages + mm->ksm_zero_pages) *PAGE_SIZE - +return (long )(mm->ksm_merging_pages +mm_ksm_zero_pages (mm)) *PAGE_SIZE - mm->ksm_rmap_items *sizeof (struct ksm_rmap_item ); }#endif /* CONFIG_PROC_FS */ @@ -3662,7 +3661,7 @@ KSM_ATTR_RO(pages_skipped);static ssize_t ksm_zero_pages_show (struct kobject *kobj,struct kobj_attribute *attr,char *buf) { -return sysfs_emit (buf,"%ld\n" , ksm_zero_pages); +return sysfs_emit (buf,"%ld\n" ,atomic_long_read (&ksm_zero_pages)); }KSM_ATTR_RO (ksm_zero_pages);@@ -3671,7 +3670,7 @@ static ssize_t general_profit_show(struct kobject *kobj, {long general_profit; - general_profit = (ksm_pages_sharing + ksm_zero_pages) *PAGE_SIZE - + general_profit = (ksm_pages_sharing +atomic_long_read (&ksm_zero_pages)) *PAGE_SIZE - ksm_rmap_items *sizeof (struct ksm_rmap_item );return sysfs_emit (buf,"%ld\n" , general_profit);Signature · 2 lines
-- 2.45.0
Chengming Zhou
reviewerRe: [PATCH v2 2/2] mm/ksm: fix ksm_zero_pages accounting
Quoted 5 lines · xu xin
Quoted 3 lines
We normally ksm_zero_pages++ in ksmd when page is merged with zero page, but ksm_zero_pages-- is done from page tables side, which can't protected by the ksmd mutex.
"cant protected" -> "can't be protected".
Right, will fix.
Quoted 4 lines
But It's better to say "where there is no any accessing protection of
ksm_zero_pages" because ksmd mutex is to protect the flag ofksm_run, not to protect the counters of KSM.
Ah, I thought the introduced ksm_zero_pages counters were protected by ksmd mutex like all other ksm counters, no? But the difference with those ksm counters is that ksm_zero_pages could be changed by the page table side operations, which can't take ksmd mutex, this is the reason why we need to use atomic variables.
Thanks.
Quoted 136 lines
Anyway, The following code looks OK to me.
Quoted 133 lines
So we can read very exceptional value of
ksm_zero_pagesin rare cases, such as -1, which is very confusing to users.Fix it by changing to use
atomic_long_t, and the same case with themm->ksm_zero_pages.Fixes e2942062e01d ("ksm: count all zero pages placed by KSM")Fixes 6080d19f0704 ("ksm: add ksm zero pages for each process")Signed-off-by Chengming Zhou <[email protected]>--- fs/proc/base.c | 2 +- include/linux/ksm.h | 17 ++++++++++++++--- include/linux/mm_types.h | 2 +- mm/ksm.c | 11 +++++------ 4 files changed, 21 insertions(+), 11 deletions(-)fs/proc/base.c +1
-1@@ -3214,7 +3214,7 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm =get_task_mm (task);if (mm) {seq_printf (m,"ksm_rmap_items %lu\n" , mm->ksm_rmap_items); -seq_printf (m,"ksm_zero_pages %lu\n" , mm->ksm_zero_pages); +seq_printf (m,"ksm_zero_pages %ld\n" ,mm_ksm_zero_pages (mm));seq_printf (m,"ksm_merging_pages %lu\n" , mm->ksm_merging_pages);seq_printf (m,"ksm_process_profit %ld\n" ,ksm_process_profit (mm));mmput (mm);include/linux/ksm.h +14
-3@@ -33,16 +33,27 @@ void __ksm_exit(struct mm_struct *mm);*/ #define is_ksm_zero_pte (pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) -extern unsigned long ksm_zero_pages; +extern atomic_long_t ksm_zero_pages; + +static inline void ksm_map_zero_page (struct mm_struct *mm) +{ +atomic_long_inc (&ksm_zero_pages); +atomic_long_inc (&mm->ksm_zero_pages); +}static inline void ksm_might_unmap_zero_page (struct mm_struct *mm,pte_t pte) {if (is_ksm_zero_pte (pte)) { - ksm_zero_pages--; - mm->ksm_zero_pages--; +atomic_long_dec (&ksm_zero_pages); +atomic_long_dec (&mm->ksm_zero_pages); } } +static inline long mm_ksm_zero_pages (struct mm_struct *mm) +{ +return atomic_long_read (&mm->ksm_zero_pages); +} +static inline int ksm_fork (struct mm_struct *mm,struct mm_struct *oldmm) {if (test_bit (MMF_VM_MERGEABLE , &oldmm->flags))include/linux/mm_types.h +1
-1@@ -985,7 +985,7 @@ struct mm_struct {* Represent how many empty pages are merged with kernel zero * pages when enabling KSM use_zero_pages. */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages;#endif /* CONFIG_KSM */ #ifdef CONFIG_LRU_GEN_WALKS_MMU struct {mm/ksm.c +5
-6@@ -296,7 +296,7 @@ static bool ksm_use_zero_pages __read_mostly;static bool ksm_smart_scan = true;/* The number of zero pages which is placed by KSM */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages =ATOMIC_LONG_INIT (0 );/* The number of pages that have been skipped due to "smart scanning" */ static unsigned long ksm_pages_skipped;@@ -1429,8 +1429,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,* the dirty bit in zero page's PTE is set. */ newpte =pte_mkdirty (pte_mkspecial (pfn_pte (page_to_pfn (kpage), vma->vm_page_prot))); - ksm_zero_pages++; - mm->ksm_zero_pages++; +ksm_map_zero_page (mm);/* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we @@ -3373,7 +3372,7 @@ static void wait_while_offlining(void)#ifdef CONFIG_PROC_FS long ksm_process_profit (struct mm_struct *mm) { -return (long )(mm->ksm_merging_pages + mm->ksm_zero_pages) *PAGE_SIZE - +return (long )(mm->ksm_merging_pages +mm_ksm_zero_pages (mm)) *PAGE_SIZE - mm->ksm_rmap_items *sizeof (struct ksm_rmap_item ); }#endif /* CONFIG_PROC_FS */ @@ -3662,7 +3661,7 @@ KSM_ATTR_RO(pages_skipped);static ssize_t ksm_zero_pages_show (struct kobject *kobj,struct kobj_attribute *attr,char *buf) { -return sysfs_emit (buf,"%ld\n" , ksm_zero_pages); +return sysfs_emit (buf,"%ld\n" ,atomic_long_read (&ksm_zero_pages)); }KSM_ATTR_RO (ksm_zero_pages);@@ -3671,7 +3670,7 @@ static ssize_t general_profit_show(struct kobject *kobj, {long general_profit; - general_profit = (ksm_pages_sharing + ksm_zero_pages) *PAGE_SIZE - + general_profit = (ksm_pages_sharing +atomic_long_read (&ksm_zero_pages)) *PAGE_SIZE - ksm_rmap_items *sizeof (struct ksm_rmap_item );return sysfs_emit (buf,"%ld\n" , general_profit);Signature · 2 lines
-- 2.45.0