mm-convert-mms-rss-stats-into-percpu_counter.patch removed from -mm tree
The quilt patch titled
Subject: mm: convert mm's rss stats into percpu_counter
has been removed from the -mm tree. Its filename was
mm-convert-mms-rss-stats-into-percpu_counter.patch
This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Shakeel Butt <[email protected]>
Subject: mm: convert mm's rss stats into percpu_counter
Date: Mon, 24 Oct 2022 05:28:41 +0000
Currently mm_struct maintains rss_stats which are updated on page fault and the unmapping codepaths. For page fault codepath the updates are cached per thread with the batch of TASK_RSS_EVENTS_THRESH which is 64. The reason for caching is performance for multithreaded applications otherwise the rss_stats updates may become hotspot for such applications.
However this optimization comes with the cost of error margin in the rss stats. The rss_stats for applications with large number of threads can be very skewed. At worst the error margin is (nr_threads * 64) and we have a lot of applications with 100s of threads, so the error margin can be very high. Internally we had to reduce TASK_RSS_EVENTS_THRESH to 32.
Recently we started seeing the unbounded errors for rss_stats for specific applications which use TCP rx0cp. It seems like vm_insert_pages() codepath does not sync rss_stats at all.
This patch converts the rss_stats into percpu_counter to convert the error margin from (nr_threads * 64) to approximately (nr_cpus ^ 2). However this conversion enable us to get the accurate stats for situations where accuracy is more important than the cpu cost.
This patch does not make such tradeoffs - we can just use percpu_counter_add_local() for the updates and percpu_counter_sum() (or percpu_counter_sync() + percpu_counter_read) for the readers. At the moment the readers are either procfs interface, oom_killer and memory reclaim which I think are not performance critical and should be ok with slow read. However I think we can make that change in a separate patch.
---
include/linux/mm.h | 26 +++-------- include/linux/mm_types.h | 7 -- include/linux/mm_types_task.h | 13 ----- include/linux/percpu_counter.h | 1 include/linux/sched.h | 3 - include/trace/events/kmem.h | 8 +-- kernel/fork.c | 16 ++++++ mm/memory.c | 73 ++++--------------------------- 8 files changed, 40 insertions(+), 107 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-convert-mms-rss-stats-into-percpu_counter +8 -18
@@ -2052,40 +2052,30 @@ static inline bool get_user_page_fast_on*/ static inline unsigned long get_mm_counter (struct mm_struct *mm,int member) { -long val =atomic_long_read (&mm->rss_stat.count[member]); - -#ifdef SPLIT_RSS_COUNTING -/* -* counter is updated in asynchronous manner and may go to minus. -* But it's never be expected number for users. -*/ -if (val <0 ) - val =0 ; -#endif -return (unsigned long )val; +return percpu_counter_read_positive (&mm->rss_stat[member]); } -void mm_trace_rss_stat (struct mm_struct *mm,int member,long count); +void mm_trace_rss_stat (struct mm_struct *mm,int member);static inline void add_mm_counter (struct mm_struct *mm,int member,long value) { -long count =atomic_long_add_return (value, &mm->rss_stat.count[member]); +percpu_counter_add (&mm->rss_stat[member], value); -mm_trace_rss_stat (mm, member, count); +mm_trace_rss_stat (mm, member); }static inline void inc_mm_counter (struct mm_struct *mm,int member) { -long count =atomic_long_inc_return (&mm->rss_stat.count[member]); +percpu_counter_inc (&mm->rss_stat[member]); -mm_trace_rss_stat (mm, member, count); +mm_trace_rss_stat (mm, member); }static inline void dec_mm_counter (struct mm_struct *mm,int member) { -long count =atomic_long_dec_return (&mm->rss_stat.count[member]); +percpu_counter_dec (&mm->rss_stat[member]); -mm_trace_rss_stat (mm, member, count); +mm_trace_rss_stat (mm, member); }/* Optimized variant when page is already known not to be PageAnon */
include/linux/mm_types.h renamed from include/linux/mm_types.h~mm-convert-mms-rss-stats-into-percpu_counter +2 -5
@@ -18,6 +18,7 @@#include <linux/page-flags-layout.h> #include <linux/workqueue.h> #include <linux/seqlock.h> +#include <linux/percpu_counter.h> #include <asm/mmu.h>
@@ -626,11 +627,7 @@ struct mm_struct {unsigned long saved_auxv[AT_VECTOR_SIZE ];/* for /proc/PID/auxv */ -/* -* Special counters, in some configurations protected by the -* page_table_lock, in other configurations by being atomic. -*/ -struct mm_rss_stat rss_stat; +struct percpu_counter rss_stat[NR_MM_COUNTERS ];struct linux_binfmt *binfmt;
include/linux/mm_types_task.h renamed from include/linux/mm_types_task.h~mm-convert-mms-rss-stats-into-percpu_counter +0 -13
@@ -36,19 +36,6 @@ enum {NR_MM_COUNTERS }; -#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU ) -#define SPLIT_RSS_COUNTING -/* per-thread cached information, */ -struct task_rss_stat { -int events;/* for synchronization threshold */ -int count[NR_MM_COUNTERS ]; -}; -#endif /* USE_SPLIT_PTE_PTLOCKS */ - -struct mm_rss_stat { -atomic_long_t count[NR_MM_COUNTERS ]; -}; -struct page_frag {struct page *page;#if (BITS_PER_LONG >32 ) || (PAGE_SIZE >=65536 )
include/linux/percpu_counter.h renamed from include/linux/percpu_counter.h~mm-convert-mms-rss-stats-into-percpu_counter +0 -1
@@ -13,7 +13,6 @@#include <linux/threads.h> #include <linux/percpu.h> #include <linux/types.h> -#include <linux/gfp.h> /* percpu_counter batch for local add or sub */ #define PERCPU_COUNTER_LOCAL_BATCH INT_MAX
include/linux/sched.h renamed from include/linux/sched.h~mm-convert-mms-rss-stats-into-percpu_counter +0 -3
@@ -870,9 +870,6 @@ struct task_struct {struct mm_struct *mm;struct mm_struct *active_mm; -#ifdef SPLIT_RSS_COUNTING -struct task_rss_stat rss_stat; -#endif int exit_state;int exit_code;int exit_signal;
include/trace/events/kmem.h renamed from include/trace/events/kmem.h~mm-convert-mms-rss-stats-into-percpu_counter +4 -4
@@ -346,10 +346,9 @@ TRACE_MM_PAGES TRACE_EVENT(rss_stat, TP_PROTO(struct mm_struct *mm, - int member, - long count), + int member), - TP_ARGS(mm, member, count), + TP_ARGS(mm, member), TP_STRUCT__entry( __field(unsigned int, mm_id)
@@ -362,7 +361,8 @@ TRACE_EVENT(rss_stat, __entry->mm_id =mm_ptr_to_hash (mm); __entry->curr = !!(current->mm == mm); __entry->member = member; - __entry->size = (count <<PAGE_SHIFT ); + __entry->size = (percpu_counter_sum_positive (&mm->rss_stat[member]) + <<PAGE_SHIFT ); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB",
kernel/fork.c renamed from kernel/fork.c~mm-convert-mms-rss-stats-into-percpu_counter +15 -1
@@ -753,7 +753,7 @@ static void check_mm(struct mm_struct *m "Please make sure 'struct resident_page_types[]' is updated as well");for (i =0 ; i <NR_MM_COUNTERS ; i++) { -long x =atomic_long_read (&mm->rss_stat.count[i]); +long x =percpu_counter_sum (&mm->rss_stat[i]);if (unlikely (x)) pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
@@ -779,6 +779,8 @@ static void check_mm(struct mm_struct *m*/ void __mmdrop (struct mm_struct *mm) { +int i; +BUG_ON (mm == &init_mm);WARN_ON_ONCE (mm == current->mm);WARN_ON_ONCE (mm == current->active_mm);
@@ -788,6 +790,9 @@ void __mmdrop(struct mm_struct *mm)check_mm (mm);put_user_ns (mm->user_ns);mm_pasid_drop (mm); + +for (i =0 ; i <NR_MM_COUNTERS ; i++) +percpu_counter_destroy (&mm->rss_stat[i]);free_mm (mm); }EXPORT_SYMBOL_GPL (__mmdrop);
@@ -1107,6 +1112,8 @@ static void mm_init_uprobes_state(structstatic struct mm_struct *mm_init (struct mm_struct *mm,struct task_struct *p,struct user_namespace *user_ns) { +int i; +mt_init_flags (&mm->mm_mt,MM_MT_FLAGS );mt_set_external_lock (&mm->mm_mt, &mm->mmap_lock);atomic_set (&mm->mm_users,1 );
@@ -1148,10 +1155,17 @@ static struct mm_struct *mm_init(structif (init_new_context (p, mm)) goto fail_nocontext; +for (i =0 ; i <NR_MM_COUNTERS ; i++) +if (percpu_counter_init (&mm->rss_stat[i],0 ,GFP_KERNEL_ACCOUNT )) + goto fail_pcpu; + mm->user_ns =get_user_ns (user_ns);lru_gen_init_mm (mm);return mm; +fail_pcpu: +while (i >0 ) +percpu_counter_destroy (&mm->rss_stat[--i]); fail_nocontext:mm_free_pgd (mm); fail_nopgd:
mm/memory.c renamed from mm/memory.c~mm-convert-mms-rss-stats-into-percpu_counter +11 -62
123 lines not on this page · raw message
_
Patches currently in -mm which might be from [email protected] are