sched: Fix performance regression introduced by mm_cid
Subsystems (3)
RFC PATCH 1/1 sched: Fix performance regression introduced by mm_cid
Introduce per-mm/cpu current concurrency id (mm_cid) to fix a PostgreSQL sysbench regression reported by Aaron Lu.
Keep track of the currently allocated mm_cid for each mm/cpu rather than freeing them immediately on context switch. This eliminates most atomic operations when context switching back and forth between threads belonging to different memory spaces in multi-threaded scenarios (many processes, each with many threads). The per-mm/per-cpu mm_cid values are serialized by their respective runqueue locks.
Thread migration is handled by introducing invocation to sched_mm_cid_migrate_from() (with source runqueue lock held) and to sched_mm_cid_migrate_to() (with destination runqueue lock held) in move_queued_task().
sched_mm_cid_migrate_from() clears the mm_cid from the task's mm per-cpu index corresponding to the source runqueue if it matches the last mm_cid observed by the migrated task. This last mm_cid value is returned as a hint to conditionally clear the mm's per-cpu mm_cid on the destination cpu.
Then, in sched_mm_cid_migrate_to(), if the last mm_cid is smaller than the mm's destination cpu current mm_cid, clear the mm's destination cpu current mm_cid. If the migrated task's mm is in use on the destination cpu, the reclaim of the mm_cid will be done lazily on the next destination cpu context switch, else it is performed immediately.
The source cpu's mm_cid is _not_ simply moved to the destination cpu on migration, because passing ownership of the mm_cid value to the destination cpu while an actively running tasks also has its own mm_cid value (in case of lazy reclaim on next context switch) would over-allocate mm_cid values beyond the number of possible cpus.
Because we want to ensure the mm_cid converges towards the smaller values as migrations happen, the prior optimization that was done when context switching between threads belonging to the same mm is removed, because it could delay the lazy release of the destination runqueue mm_cid after it has been replaced by a migration. Removing this prior optimization is not an issue performance-wise because the introduced per-mm/per-cpu mm_cid tracking also covers this more specific case.
This patch is based on v6.3-rc5 with this patch applied:
("mm: Fix memory leak on mm_init error handling")
https://lore.kernel.org/lkml/[email protected]/
--- include/linux/mm_types.h | 30 ++++++++++ include/linux/sched.h | 1 + kernel/fork.c | 8 ++- kernel/sched/core.c | 122 +++++++++++++++++++++++++++++++++++---- kernel/sched/sched.h | 83 +++++++++++++++++++------- 5 files changed, 210 insertions(+), 34 deletions(-)
include/linux/mm_types.h +30 -0
@@ -609,6 +609,14 @@ struct mm_struct {* were being concurrently updated by the updaters. */ raw_spinlock_t cid_lock; +/** +* @pcpu_cid: Per-cpu current cid. +* +* Keep track of the currently allocated mm_cid for each cpu. +* The per-cpu mm_cid values are serialized by their respective +* runqueue locks. +*/ +int __percpu *pcpu_cid;#endif #ifdef CONFIG_MMU atomic_long_t pgtables_bytes;/* size of all page tables */
@@ -872,6 +880,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi, }#ifdef CONFIG_SCHED_MM_CID +/* Accessor for struct mm_struct's cidmask. */ static inline cpumask_t *mm_cidmask (struct mm_struct *mm) {
@@ -885,16 +894,37 @@ static inline cpumask_t *mm_cidmask(struct mm_struct *mm)static inline void mm_init_cid (struct mm_struct *mm) { +int i; +raw_spin_lock_init (&mm->cid_lock); +for_each_possible_cpu (i) + *per_cpu_ptr (mm->pcpu_cid, i) =-1 ;cpumask_clear (mm_cidmask (mm)); } +static inline int mm_alloc_cid (struct mm_struct *mm) +{ + mm->pcpu_cid =alloc_percpu (int); +if (!mm->pcpu_cid) +return -ENOMEM ; +mm_init_cid (mm); +return 0 ; +} + +static inline void mm_destroy_cid (struct mm_struct *mm) +{ +free_percpu (mm->pcpu_cid); + mm->pcpu_cid =NULL ; +} +static inline unsigned int mm_cid_size (void ) {return cpumask_size (); }#else /* CONFIG_SCHED_MM_CID */ static inline void mm_init_cid (struct mm_struct *mm) { } +static inline int mm_alloc_cid (struct mm_struct *mm) {return 0 ; } +static inline void mm_destroy_cid (struct mm_struct *mm) { }static inline unsigned int mm_cid_size (void ) {return 0 ;
include/linux/sched.h +1 -0
@@ -1313,6 +1313,7 @@ struct task_struct {#ifdef CONFIG_SCHED_MM_CID int mm_cid;/* Current cid in mm */ +int last_mm_cid;/* Most recent cid in mm */ int mm_cid_active;/* Whether cid bitmap is active */ #endif
kernel/fork.c +7 -1
@@ -790,6 +790,7 @@ void __mmdrop(struct mm_struct *mm)check_mm (mm);put_user_ns (mm->user_ns);mm_pasid_drop (mm); +mm_destroy_cid (mm);for (i =0 ; i <NR_MM_COUNTERS ; i++)percpu_counter_destroy (&mm->rss_stat[i]);
@@ -1054,6 +1055,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)#ifdef CONFIG_SCHED_MM_CID tsk->mm_cid =-1 ; + tsk->last_mm_cid =-1 ; tsk->mm_cid_active =0 ;#endif return tsk;
@@ -1159,18 +1161,22 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,if (init_new_context (p, mm)) goto fail_nocontext; +if (mm_alloc_cid (mm)) + goto fail_cid; +for (i =0 ; i <NR_MM_COUNTERS ; i++)if (percpu_counter_init (&mm->rss_stat[i],0 ,GFP_KERNEL_ACCOUNT )) goto fail_pcpu; mm->user_ns =get_user_ns (user_ns);lru_gen_init_mm (mm); -mm_init_cid (mm);return mm; fail_pcpu:while (i >0 )percpu_counter_destroy (&mm->rss_stat[--i]); +mm_destroy_cid (mm); +fail_cid:destroy_context (mm); fail_nocontext:mm_free_pgd (mm);
kernel/sched/core.c +110 -12
171 lines not on this page · raw message
kernel/sched/sched.h +62 -21
117 lines not on this page · raw message
Signature · 1 line
2.25.1
On 2023-04-03 14:13, Mathieu Desnoyers wrote:
[...]
Quoted 11 lines
kernel/sched/core.c +11
-0+/* +* Migration to dst cpu. Called with dst_rq lock held. +*/ +void sched_mm_cid_migrate_to (struct rq *dst_rq,struct task_struct *t,int src_cid) +{ +struct mm_struct *mm = t->mm; +int dst_cid, *dst_pcpu_cid; + +lockdep_assert_rq_held (dst_rq); + +if (!mm || src_cid ==-1 )
kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11458
Quoted 10 lines
kernel/sched/core.c +10
-0+return ; + + dst_pcpu_cid =per_cpu_ptr (mm->pcpu_cid,cpu_of (dst_rq)); + +/* +* If destination cpu cid is greater than the source cpu cid, unset it +* so it can be reallocated. +*/ + dst_cid = *dst_pcpu_cid; +if (dst_cid ==-1 || dst_cid < src_cid)
Small detail: I plan to change this from "dst_cid < src_cid" to "dst_cid <= src_cid" in my next version of the patch to handle the unlikely case where a task would be migrated back and forth between two runqueues without being scheduled. It would be possible that the task's last_mm_cid is equal to the dst_cid here, in which case it would be better to leave the mm's destination cpu cid set.
kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11467
Quoted 9 lines
kernel/sched/core.c +9
-0+return ; + *dst_pcpu_cid =-1 ; +/* +* Put dst_cid if it is not currently in use, else it will be lazy put +* on the next context switch. +*/ +if (dst_rq->curr->mm != mm) +__mm_cid_put (mm, dst_cid); +}
[...]
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Quoted 22 lines · Mathieu Desnoyers
On 2023-04-03 14:13, Mathieu Desnoyers wrote:
[...]Quoted 20 lines
kernel/sched/core.c +20
-0+/* +* Migration to dst cpu. Called with dst_rq lock held. +*/ +void sched_mm_cid_migrate_to (struct rq *dst_rq,struct task_struct *t,int src_cid) +{ +struct mm_struct *mm = t->mm; +int dst_cid, *dst_pcpu_cid; + +lockdep_assert_rq_held (dst_rq); + +if (!mm || src_cid ==-1 ) +return ; + + dst_pcpu_cid =per_cpu_ptr (mm->pcpu_cid,cpu_of (dst_rq)); + +/* +* If destination cpu cid is greater than the source cpu cid, unset it +* so it can be reallocated. +*/ + dst_cid = *dst_pcpu_cid;
Quoted 9 lines
kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11458
Quoted 1 line
kernel/sched/core.c +1
-0+if (dst_cid ==-1 || dst_cid < src_cid)Small detail: I plan to change this from "
dst_cid<src_cid" to "dst_cid<=src_cid" in my next version of the patch to handle the unlikely case where a task would be migrated back and forth between two runqueues without being scheduled. It would be possible that the task'slast_mm_cidis equal to thedst_cidhere, in which case it would be better to leave the mm's destination cpu cid set.
This patch is still good regarding lock contention. (I applied the above small change while testing)
Quoted 9 lines
kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11467
Quoted 9 lines
kernel/sched/core.c +9
-0+return ; + *dst_pcpu_cid =-1 ; +/* +* Put dst_cid if it is not currently in use, else it will be lazy put +* on the next context switch. +*/ +if (dst_rq->curr->mm != mm) +__mm_cid_put (mm, dst_cid); +}
Thanks,
Aaron
Peter Zijlstra
maintainerSorry for being a little late to the party.
Quoted 6 lines · Mathieu Desnoyers
include/linux/mm_types.h +0
-0@@ -609,6 +609,14 @@ struct mm_struct {* were being concurrently updated by the updaters.
include/linux/mm_types.h struct mm_struct { · line 619
Quoted 10 lines
include/linux/mm_types.h +8
-0*/ raw_spinlock_t cid_lock; +/** +* @pcpu_cid: Per-cpu current cid. +* +* Keep track of the currently allocated mm_cid for each cpu. +* The per-cpu mm_cid values are serialized by their respective +* runqueue locks. +*/ +int __percpu *pcpu_cid;
This *might* be a problem... Consider running *many* single threaded processes, this could exhaust the limited per-cpu storage (32bit) or use excessive memory on large CPU systems.
Consider having to allocate per-cpu storage on a 4K CPU system while only running single threaded processes -- but *LOTS* of them..
Peter Zijlstra
maintainerQuoted 26 lines · Peter Zijlstra
Sorry for being a little late to the party.
Quoted 6 lines · Mathieu Desnoyers
include/linux/mm_types.h +0
-0@@ -609,6 +609,14 @@ struct mm_struct {* were being concurrently updated by the updaters. include/linux/mm_types.h struct mm_struct { · line 619
Quoted 10 lines
include/linux/mm_types.h +8
-0*/ raw_spinlock_t cid_lock; +/** +* @pcpu_cid: Per-cpu current cid. +* +* Keep track of the currently allocated mm_cid for each cpu. +* The per-cpu mm_cid values are serialized by their respective +* runqueue locks. +*/ +int __percpu *pcpu_cid;This *might* be a problem... Consider running *many* single threaded processes, this could exhaust the limited per-cpu storage (32bit) or use excessive memory on large CPU systems.
Quoted 2 lines
Consider having to allocate per-cpu storage on a 4K CPU system while only running single threaded processes -- but *LOTS* of them..
Ah, mm_struct::rss_stat[] beat us and set precedent, so not our problem I suppose :-)
Quoted 27 lines · Peter Zijlstra
Quoted 26 lines · Peter Zijlstra
Sorry for being a little late to the party.
Quoted 6 lines · Mathieu Desnoyers
include/linux/mm_types.h +0
-0@@ -609,6 +609,14 @@ struct mm_struct {* were being concurrently updated by the updaters. include/linux/mm_types.h struct mm_struct { · line 619
Quoted 10 lines
include/linux/mm_types.h +8
-0*/ raw_spinlock_t cid_lock; +/** +* @pcpu_cid: Per-cpu current cid. +* +* Keep track of the currently allocated mm_cid for each cpu. +* The per-cpu mm_cid values are serialized by their respective +* runqueue locks. +*/ +int __percpu *pcpu_cid;This *might* be a problem... Consider running *many* single threaded processes, this could exhaust the limited per-cpu storage (32bit) or use excessive memory on large CPU systems.
Quoted 5 lines
Quoted 2 lines
Consider having to allocate per-cpu storage on a 4K CPU system while only running single threaded processes -- but *LOTS* of them..
Ah, mm_struct::rss_stat[] beat us and set precedent, so not our problem I suppose :-)
Yes, exactly. When I've seen that mm_struct::rss_stat[] did not care about having per-mm/per-cpu data, I went the same way.
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Peter Zijlstra
maintainerQuoted 12 lines · Mathieu Desnoyers
void
sched_mm_cid_exit_signals(struct task_struct *t){kernel/sched/core.c +4
-2struct mm_struct *mm = t->mm; -unsigned long flags; +struct rq_flags rf; +struct rq *rq;if (!mm)return ; -local_irq_save (flags); + +preempt_disable ();
kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11486
Quoted 10 lines
kernel/sched/core.c +5
-2+ rq =this_rq (); +rq_lock_irqsave (rq, &rf);mm_cid_put (mm, t->mm_cid); - t->mm_cid =-1 ; + t->last_mm_cid = t->mm_cid =-1 ; t->mm_cid_active =0 ; -local_irq_restore (flags); +rq_unlock_irqrestore (rq, &rf); +preempt_enable (); }
FWIW a *slightly* cheaper form is:
preempt_disable(); rq = this_rq(); rq_lock_irqsave(rq, &rf); preempt_enable_noresched(); /* holding spinlock */ ... rq_unlock_irqrestore(rq, &rf);
Quoted 26 lines · Peter Zijlstra
Quoted 12 lines · Mathieu Desnoyers
void sched_mm_cid_exit_signals(struct task_struct *t) { struct mm_struct *mm = t->mm;kernel/sched/core.c +4
-2-unsigned long flags; +struct rq_flags rf; +struct rq *rq;if (!mm)return ; -local_irq_save (flags); + +preempt_disable ();kernel/sched/core.c void call_trace_sched_update_nr_running(struct rq *rq, int count) · line 11486
Quoted 10 lines
kernel/sched/core.c +5
-2+ rq =this_rq (); +rq_lock_irqsave (rq, &rf);mm_cid_put (mm, t->mm_cid); - t->mm_cid =-1 ; + t->last_mm_cid = t->mm_cid =-1 ; t->mm_cid_active =0 ; -local_irq_restore (flags); +rq_unlock_irqrestore (rq, &rf); +preempt_enable (); }FWIW a *slightly* cheaper form is:
Quoted 6 lines
preempt_disable(); rq = this_rq(); rq_lock_irqsave(rq, &rf); preempt_enable_noresched(); /* holding spinlock */ ... rq_unlock_irqrestore(rq, &rf);
Good point, updated for next round.
Thanks,
Mathieu
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Peter Zijlstra
maintainerQuoted 15 lines · Mathieu Desnoyers
kernel/sched/core.c +3
-0@@ -2326,16 +2326,20 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)static struct rq *move_queued_task (struct rq *rq,struct rq_flags *rf,struct task_struct *p,int new_cpu) { +int cid; +lockdep_assert_rq_held (rq);deactivate_task (rq, p,DEQUEUE_NOCLOCK );set_task_cpu (p, new_cpu); + cid =sched_mm_cid_migrate_from (rq, p);
kernel/sched/core.c static inline bool is_cpu_allowed(struct task_struct *p, int cpu) · line 2345
Quoted 10 lines
kernel/sched/core.c +1
-0rq_unlock (rq, rf); rq =cpu_rq (new_cpu);rq_lock (rq, rf);WARN_ON_ONCE (task_cpu (p) != new_cpu); +sched_mm_cid_migrate_to (rq, p, cid);activate_task (rq, p,0 );check_preempt_curr (rq, p,0 );
I can't find more uses? Yet there are a ton more ways to actually migrate tasks.