sched: Store restrict_cpus_allowed_ptr() call state
PATCH 1/1 sched: Store restrict_cpus_allowed_ptr() call state
The user_cpus_ptr field was originally added by commit b90ca8badbd1 ("sched: Introduce task_struct::user_cpus_ptr to track requested affinity"). It was used only by arm64 arch due to possible asymmetric CPU setup.
Since commit 8f9ea86fdf99 ("sched: Always preserve the user requested cpumask"), task_struct::user_cpus_ptr is repurposed to store user requested cpu affinity specified in the sched_setaffinity().
This results in a performance regression in an arm64 system when booted with "allow_mismatched_32bit_el0" on the command-line. The arch code will (amongst other things) calls force_compatible_cpus_allowed_ptr() and relax_compatible_cpus_allowed_ptr() when exec()'ing a 32-bit or a 64-bit task respectively. Now a call to relax_compatible_cpus_allowed_ptr() will always result in a __sched_setaffinity() call whether there is a previous force_compatible_cpus_allowed_ptr() call or not.
In order to fix this regression, a new scheduler flag task_struct::cpus_allowed_restricted is now added to track if force_compatible_cpus_allowed_ptr() has been called before or not. This patch also updates the comments in force_compatible_cpus_allowed_ptr() and relax_compatible_cpus_allowed_ptr() and handles their interaction with sched_setaffinity().
--- include/linux/sched.h | 3 +++ kernel/sched/core.c | 55 ++++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 2 ++ 3 files changed, 51 insertions(+), 9 deletions(-)
include/linux/sched.h +3 -0
@@ -886,6 +886,9 @@ struct task_struct { unsigned sched_contributes_to_load:1; unsigned sched_migrated:1; +/* restrict_cpus_allowed_ptr() bit, serialized by scheduler locks */ + unsigned cpus_allowed_restricted:1; +/* Force alignment to the next boundary: */ unsigned :0;
kernel/sched/core.c +46 -9
95 lines not on this page · raw message
kernel/sched/sched.h +2 -0
@@ -2293,6 +2293,8 @@ extern struct task_struct *pick_next_task_idle(struct rq *rq);#define SCA_MIGRATE_DISABLE 0x02#define SCA_MIGRATE_ENABLE 0x04#define SCA_USER 0x08 +#define SCA_CLR_RESTRICT 0x10 +#define SCA_SET_RESTRICT 0x20#ifdef CONFIG_SMP
Signature · 1 line
2.31.1
Quoted 72 lines · Waiman Long
The
user_cpus_ptrfield was originally added by commitb90ca8badbd1("sched: Introducetask_struct::user_cpus_ptrto track requested affinity"). It was used only by arm64 arch due to possible asymmetric CPU setup.Since commit
8f9ea86fdf99("sched: Always preserve the user requested cpumask"),task_struct::user_cpus_ptris repurposed to store user requested cpu affinity specified in thesched_setaffinity().This results in a performance regression in an arm64 system when booted with "
allow_mismatched_32bit_el0" on the command-line. The arch code will (amongst other things) callsforce_compatible_cpus_allowed_ptr()andrelax_compatible_cpus_allowed_ptr()when exec()'ing a 32-bit or a 64-bit task respectively. Now a call torelax_compatible_cpus_allowed_ptr()will always result in a__sched_setaffinity()call whether there is a previousforce_compatible_cpus_allowed_ptr()call or not.In order to fix this regression, a new scheduler flag
task_struct::cpus_allowed_restrictedis now added to track ifforce_compatible_cpus_allowed_ptr()has been called before or not. This patch also updates the comments inforce_compatible_cpus_allowed_ptr()andrelax_compatible_cpus_allowed_ptr()and handles their interaction withsched_setaffinity().Fixes 8f9ea86fdf99 ("sched: Always preserve the user requested cpumask")Reported-by Will Deacon <[email protected]>Signed-off-by Waiman Long <[email protected]>--- include/linux/sched.h | 3 +++ kernel/sched/core.c | 55 ++++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 2 ++ 3 files changed, 51 insertions(+), 9 deletions(-)include/linux/sched.h +3
-0@@ -886,6 +886,9 @@ struct task_struct { unsigned sched_contributes_to_load:1; unsigned sched_migrated:1; +/* restrict_cpus_allowed_ptr() bit, serialized by scheduler locks */ + unsigned cpus_allowed_restricted:1; +/* Force alignment to the next boundary: */ unsigned :0;kernel/sched/core.c +17
-0@@ -2999,15 +2999,49 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,struct rq *rq; rq =task_rq_lock (p, &rf); + +if (ctx->flags & (SCA_CLR_RESTRICT |SCA_SET_RESTRICT )) { + p->cpus_allowed_restricted =0 ; + }else if (p->cpus_allowed_restricted) { +/* +* If force_compatible_cpus_allowed_ptr() has been called, +* we can't extend cpumask to beyond what is in cpus_mask. +*/ +if (!cpumask_and (rq->scratch_mask, ctx->new_mask, + &p->cpus_mask)) { +task_rq_unlock (rq, p, &rf); +return -EINVAL ; + } + +/* +* In the unlikely event that sched_setaffinity() has been +* called on a cpu with cpus_allowed_restricted flag set,
kernel/sched/core.c static int __set_cpus_allowed_ptr(struct task_struct *p, · line 3028
Quoted 10 lines
kernel/sched/core.c +10
-0+* we have to store in user_cpus_ptr a restricted version +* of cpu affinity mask that won't be restored when +* relax_compatible_cpus_allowed_ptr() is called. +* +* Note that we don't need to do further user_cpus_ptr +* masking below as cpus_mask should be a subset of +* user_cpus_ptr if set. +*/ +if (ctx->flags &SCA_USER ) +cpumask_copy ((struct cpumask *)ctx->new_mask, rq->scratch_mask);
Sorry, that part is wrong. I forgot there were two masks in ctx. Will send out a v2.
Cheers,
Longman
Quoted 82 lines
kernel/sched/core.c +19
-9+else + ctx->new_mask = rq->scratch_mask; + } +/* * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* -* flags are set. +* flags are set or when cpus_allowed_restricted flag has been set. */ -if (p->user_cpus_ptr && +if (p->user_cpus_ptr && !p->cpus_allowed_restricted && !(ctx->flags & (SCA_USER |SCA_MIGRATE_ENABLE |SCA_MIGRATE_DISABLE )) &&cpumask_and (rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) ctx->new_mask = rq->scratch_mask; +if (ctx->flags &SCA_SET_RESTRICT ) + p->cpus_allowed_restricted =1 ; +return __set_cpus_allowed_ptr_locked (p, ctx, rq, &rf); }@@ -3025,8 +3059,8 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);/* * Change a given task's CPU affinity to the intersection of its current * affinity mask and @subset_mask, writing the resulting mask to @new_mask. -* If user_cpus_ptr is defined, use it as the basis for restricting CPU -* affinity or use cpu_online_mask instead. +* The cpus_allowed_restricted bit is set to indicate to a later +* relax_compatible_cpus_allowed_ptr() call to relax the cpumask. * * If the resulting mask is empty, leave the affinity unchanged and return * -EINVAL. @@ -3037,7 +3071,7 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p, {struct affinity_context ac = { .new_mask = new_mask, - .flags =0 , + .flags =SCA_SET_RESTRICT , };struct rq_flags rf;struct rq *rq;@@ -3069,9 +3103,8 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,/* * Restrict the CPU affinity of task @p so that it is a subset of -* task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the -* old affinity mask. If the resulting mask is empty, we warn and walk -* up the cpuset hierarchy until we find a suitable mask. +* task_cpu_possible_mask(). If the resulting mask is empty, we warn +* and walk up the cpuset hierarchy until we find a suitable mask. */ void force_compatible_cpus_allowed_ptr (struct task_struct *p) {@@ -3126,10 +3159,14 @@ void relax_compatible_cpus_allowed_ptr(struct task_struct *p) {struct affinity_context ac = { .new_mask =task_user_cpus (p), - .flags =0 , + .flags =SCA_CLR_RESTRICT , };int ret; +/* Return if no previous force_compatible_cpus_allowed_ptr() call */ +if (!data_race (p->cpus_allowed_restricted)) +return ; +/* * Try to restore the old affinity mask with __sched_setaffinity(). * Cpuset masking will be done there too. kernel/sched/sched.h +2
-0@@ -2293,6 +2293,8 @@ extern struct task_struct *pick_next_task_idle(struct rq *rq);#define SCA_MIGRATE_DISABLE 0x02#define SCA_MIGRATE_ENABLE 0x04#define SCA_USER 0x08 +#define SCA_CLR_RESTRICT 0x10 +#define SCA_SET_RESTRICT 0x20#ifdef CONFIG_SMP