sched: Use lock guards, wave 2
Subsystems (2)
Peter Zijlstra
maintainercover sched: Use lock guards, wave 2
Hi, as promised, here is the second part of the SBRM patches.
Now that we're a little used to them, these are a little more involved.
New in this series, over the guard()() and scoped_guard () usage, is CLASS().
CLASS() is very like guard() except it also takes a name argument. So where guard() instantiates an 'anonymous' variable, CLASS() instantiates an explicitly named one.
CLASS(task_rq_lock, rq_guard)(p);
which can be read like:
class_task_rq_lock_t rq_guard __cleanup(class_task_rq_lock_destructor) =
class_task_rq_lock_constructor(p);
[[ Notable: guard(name) := CLASS(name, __UNIQUE_ID(guard)) ]]
The advantage in this specific case is that the guard object of task_rq_lock will contain the 'struct rq *' which we'll need.
Another use of CLASS() is:
CLASS(find_get_task, p)(pid);
Where 'p' is of 'struct task_struct *' type and the constructor does the pid->task_struct lookup and the destructor does put_task_struct.
Again, mostly aimed at getting rid of error gotos. The final patch is a bunch of random conversions that happened while I was poking around and playing with things.
Also available at:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git sched/guards2
Next in line is kernel/events/core.c, but I might not bore all of you with those patches :-)
Peter Zijlstra
maintainerPATCH 1/8 sched: Simplify set_user_nice()
Use guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 13 ++++++------- kernel/sched/sched.h | 5 +++++ 2 files changed, 11 insertions(+), 7 deletions(-)
kernel/sched/core.c +6 -7
@@ -7119,9 +7119,8 @@ static inline int rt_effective_prio(struvoid set_user_nice (struct task_struct *p,long nice) {bool queued, running; -int old_prio; -struct rq_flags rf;struct rq *rq; +int old_prio;if (task_nice (p) == nice || nice <MIN_NICE || nice >MAX_NICE )return ;
@@ -7129,7 +7128,9 @@ void set_user_nice(struct task_struct *p* We have to be careful, if called from sys_setpriority(), * the task might be in the middle of scheduling on another CPU. */ - rq =task_rq_lock (p, &rf); +CLASS (task_rq_lock, rq_guard)(p); + rq = rq_guard.rq; +update_rq_clock (rq);/*
@@ -7140,8 +7141,9 @@ void set_user_nice(struct task_struct *p*/ if (task_has_dl_policy (p) ||task_has_rt_policy (p)) { p->static_prio =NICE_TO_PRIO (nice); - goto out_unlock; +return ; } + queued =task_on_rq_queued (p); running =task_current (rq, p);if (queued)
@@ -7164,9 +7166,6 @@ void set_user_nice(struct task_struct *p* lowered its priority, then reschedule its CPU: */ p->sched_class->prio_changed (rq, p, old_prio); - -out_unlock: -task_rq_unlock (rq, p, &rf); }EXPORT_SYMBOL (set_user_nice);
kernel/sched/sched.h +5 -0
@@ -1630,6 +1630,11 @@ task_rq_unlock(struct rq *rq, struct tasraw_spin_unlock_irqrestore (&p->pi_lock, rf->flags); } +DEFINE_LOCK_GUARD_1(task_rq_lock, struct task_struct, + _T->rq =task_rq_lock (_T->lock, &_T->rf), +task_rq_unlock (_T->rq, _T->lock, &_T->rf), + struct rq *rq; struct rq_flags rf) + static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) __acquires(rq->lock)
Peter Zijlstra
maintainerPATCH 2/8 sched: Simplify syscalls
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 154 ++++++++++++++++++++++------------------------------ 1 file changed, 68 insertions(+), 86 deletions(-)
kernel/sched/core.c +68 -86
@@ -7425,6 +7425,21 @@ static struct task_struct *find_process_return pid ?find_task_by_vpid (pid) : current; } +static struct task_struct *find_get_task (pid_t pid) +{ +struct task_struct *p; +guard (rcu)(); + + p =find_process_by_pid (pid); +if (likely (p)) +get_task_struct (p); + +return p; +} + +DEFINE_CLASS(find_get_task, struct task_struct *, if (_T) put_task_struct(_T), + find_get_task(pid), pid_t pid) +/* * sched_setparam() passes in -1 for its policy, to let the functions * it calls know not to change it.
@@ -7462,14 +7477,11 @@ static void __setscheduler_params(structstatic bool check_same_owner (struct task_struct *p) {const struct cred *cred =current_cred (), *pcred; -bool match; +guard (rcu)(); -rcu_read_lock (); pcred =__task_cred (p); - match = (uid_eq (cred->euid, pcred->euid) || -uid_eq (cred->euid, pcred->uid)); -rcu_read_unlock (); -return match; +return (uid_eq (cred->euid, pcred->euid) || +uid_eq (cred->euid, pcred->uid)); }/*
@@ -7873,27 +7885,17 @@ static int do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) { struct sched_param lparam; -struct task_struct *p; -int retval;if (!param || pid <0 )return -EINVAL ;if (copy_from_user (&lparam, param,sizeof (struct sched_param )))return -EFAULT ; -rcu_read_lock (); - retval = -ESRCH ; - p =find_process_by_pid (pid); -if (likely (p)) -get_task_struct (p); -rcu_read_unlock (); - -if (likely (p)) { - retval =sched_setscheduler (p, policy, &lparam); -put_task_struct (p); - } +CLASS (find_get_task, p)(pid); +if (!p) +return -ESRCH ; -return retval; +return sched_setscheduler (p, policy, &lparam); }/*
@@ -7989,7 +7991,6 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pi unsigned int, flags) {struct sched_attr attr; -struct task_struct *p;int retval;if (!uattr || pid <0 || flags)
@@ -8004,21 +8005,14 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, piif (attr.sched_flags &SCHED_FLAG_KEEP_POLICY ) attr.sched_policy =SETPARAM_POLICY ; -rcu_read_lock (); - retval = -ESRCH ; - p =find_process_by_pid (pid); -if (likely (p)) -get_task_struct (p); -rcu_read_unlock (); +CLASS (find_get_task, p)(pid); +if (!p) +return -ESRCH ; -if (likely (p)) { -if (attr.sched_flags &SCHED_FLAG_KEEP_PARAMS ) -get_params (p, &attr); - retval =sched_setattr (p, &attr); -put_task_struct (p); - } +if (attr.sched_flags &SCHED_FLAG_KEEP_PARAMS ) +get_params (p, &attr); -return retval; +return sched_setattr (p, &attr); }/**
@@ -8036,16 +8030,17 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_if (pid <0 )return -EINVAL ; - retval = -ESRCH ; -rcu_read_lock (); +guard (rcu)(); p =find_process_by_pid (pid); -if (p) { - retval =security_task_getscheduler (p); -if (!retval) - retval = p->policy - | (p->sched_reset_on_fork ?SCHED_RESET_ON_FORK :0 ); +if (!p) +return -ESRCH ; + + retval =security_task_getscheduler (p); +if (!retval) { + retval = p->policy; +if (p->sched_reset_on_fork) + retval |=SCHED_RESET_ON_FORK ; } -rcu_read_unlock ();return retval; }
@@ -8066,30 +8061,23 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pif (!param || pid <0 )return -EINVAL ; -rcu_read_lock (); - p =find_process_by_pid (pid); - retval = -ESRCH ; -if (!p) - goto out_unlock; +scoped_guard (rcu) { + p =find_process_by_pid (pid); +if (!p) +return -ESRCH ; - retval =security_task_getscheduler (p); -if (retval) - goto out_unlock; + retval =security_task_getscheduler (p); +if (retval) +return retval; -if (task_has_rt_policy (p)) - lp.sched_priority = p->rt_priority; -rcu_read_unlock (); +if (task_has_rt_policy (p)) + lp.sched_priority = p->rt_priority; + }/* * This one might sleep, we cannot do it with a spinlock held ... */ - retval =copy_to_user (param, &lp,sizeof (*param)) ? -EFAULT :0 ; - -return retval; - -out_unlock: -rcu_read_unlock (); -return retval; +return copy_to_user (param, &lp,sizeof (*param)) ? -EFAULT :0 ; }/*
@@ -8149,39 +8137,33 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pi usize < SCHED_ATTR_SIZE_VER0 || flags)return -EINVAL ; -rcu_read_lock (); - p =find_process_by_pid (pid); - retval = -ESRCH ; -if (!p) - goto out_unlock; +scoped_guard (rcu) { + p =find_process_by_pid (pid); +if (!p) +return -ESRCH ; - retval =security_task_getscheduler (p); -if (retval) - goto out_unlock; + retval =security_task_getscheduler (p); +if (retval) +return retval; - kattr.sched_policy = p->policy; -if (p->sched_reset_on_fork) - kattr.sched_flags |=SCHED_FLAG_RESET_ON_FORK ; -get_params (p, &kattr); - kattr.sched_flags &=SCHED_FLAG_ALL ; + kattr.sched_policy = p->policy; +if (p->sched_reset_on_fork) + kattr.sched_flags |=SCHED_FLAG_RESET_ON_FORK ; +get_params (p, &kattr); + kattr.sched_flags &=SCHED_FLAG_ALL ;#ifdef CONFIG_UCLAMP_TASK -/* -* This could race with another potential updater, but this is fine -* because it'll correctly read the old or the new value. We don't need -* to guarantee who wins the race as long as it doesn't return garbage. -*/ - kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN ].value; - kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX ].value; +/* +* This could race with another potential updater, but this is fine +* because it'll correctly read the old or the new value. We don't need +* to guarantee who wins the race as long as it doesn't return garbage. +*/ + kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN ].value; + kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX ].value;#endif - -rcu_read_unlock (); + }return sched_attr_copy_to_user (uattr, &kattr, usize); - -out_unlock: -rcu_read_unlock (); -return retval; }#ifdef CONFIG_SMP
Peter Zijlstra
maintainerPATCH 3/8 sched: Simplify sched_{set,get}affinity()
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 53 +++++++++++++--------------------------------------- 1 file changed, 14 insertions(+), 39 deletions(-)
kernel/sched/core.c +14 -39
@@ -8258,39 +8258,24 @@ long sched_setaffinity(pid_t pid, const {struct affinity_context ac;struct cpumask *user_mask; -struct task_struct *p;int retval; -rcu_read_lock (); - - p =find_process_by_pid (pid); -if (!p) { -rcu_read_unlock (); +CLASS (find_get_task, p)(pid); +if (!p)return -ESRCH ; - } -/* Prevent p going away */ -get_task_struct (p); -rcu_read_unlock (); - -if (p->flags &PF_NO_SETAFFINITY ) { - retval = -EINVAL ; - goto out_put_task; - } +if (p->flags &PF_NO_SETAFFINITY ) +return -EINVAL ;if (!check_same_owner (p)) { -rcu_read_lock (); -if (!ns_capable (__task_cred (p)->user_ns,CAP_SYS_NICE )) { -rcu_read_unlock (); - retval = -EPERM ; - goto out_put_task; - } -rcu_read_unlock (); +guard (rcu)(); +if (!ns_capable (__task_cred (p)->user_ns,CAP_SYS_NICE )) +return -EPERM ; } retval =security_task_setscheduler (p);if (retval) - goto out_put_task; +return retval;/* * With non-SMP configs, user_cpus_ptr/user_mask isn't used and
@@ -8300,8 +8285,7 @@ long sched_setaffinity(pid_t pid, constif (user_mask) {cpumask_copy (user_mask, in_mask); }else if (IS_ENABLED (CONFIG_SMP )) { - retval = -ENOMEM ; - goto out_put_task; +return -ENOMEM ; } ac = (struct affinity_context ){
@@ -8313,8 +8297,6 @@ long sched_setaffinity(pid_t pid, const retval =__sched_setaffinity (p, &ac);kfree (ac.user_mask); -out_put_task: -put_task_struct (p);return retval; }
@@ -8356,28 +8338,21 @@ SYSCALL_DEFINE3(sched_setaffinity, pid_tlong sched_getaffinity (pid_t pid,struct cpumask *mask) {struct task_struct *p; -unsigned long flags;int retval; -rcu_read_lock (); - - retval = -ESRCH ; +guard (rcu)(); p =find_process_by_pid (pid);if (!p) - goto out_unlock; +return -ESRCH ; retval =security_task_getscheduler (p);if (retval) - goto out_unlock; +return retval; -raw_spin_lock_irqsave (&p->pi_lock, flags); +guard (raw_spinlock_irqsave)(&p->pi_lock);cpumask_and (mask, &p->cpus_mask, cpu_active_mask); -raw_spin_unlock_irqrestore (&p->pi_lock, flags); -out_unlock: -rcu_read_unlock (); - -return retval; +return 0 ; }/**
Peter Zijlstra
maintainerPATCH 4/8 sched: Simplify yield_to()
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 73 ++++++++++++++++++++++------------------------------ 1 file changed, 32 insertions(+), 41 deletions(-)
kernel/sched/core.c +32 -41
@@ -8799,55 +8799,46 @@ int __sched yield_to(struct task_struct {struct task_struct *curr = current;struct rq *rq, *p_rq; -unsigned long flags;int yielded =0 ; -local_irq_save (flags); - rq =this_rq (); +scoped_guard (irqsave) { + rq =this_rq (); again: - p_rq =task_rq (p); -/* -* If we're the only runnable task on the rq and target rq also -* has only one task, there's absolutely no point in yielding. -*/ -if (rq->nr_running ==1 && p_rq->nr_running ==1 ) { - yielded = -ESRCH ; - goto out_irq; - } - -double_rq_lock (rq, p_rq); -if (task_rq (p) != p_rq) { -double_rq_unlock (rq, p_rq); - goto again; - } - -if (!curr->sched_class->yield_to_task) - goto out_unlock; - -if (curr->sched_class != p->sched_class) - goto out_unlock; - -if (task_on_cpu (p_rq, p) || !task_is_running (p)) - goto out_unlock; - - yielded = curr->sched_class->yield_to_task (rq, p); -if (yielded) { -schedstat_inc (rq->yld_count); + p_rq =task_rq (p);/* -* Make p's CPU reschedule; pick_next_entity takes care of -* fairness. +* If we're the only runnable task on the rq and target rq also +* has only one task, there's absolutely no point in yielding. */ -if (preempt && rq != p_rq) -resched_curr (p_rq); - } +if (rq->nr_running ==1 && p_rq->nr_running ==1 ) +return -ESRCH ; -out_unlock: -double_rq_unlock (rq, p_rq); -out_irq: -local_irq_restore (flags); +guard (double_rq_lock)(rq, p_rq); +if (task_rq (p) != p_rq) + goto again; + +if (!curr->sched_class->yield_to_task) +return 0 ; + +if (curr->sched_class != p->sched_class) +return 0 ; + +if (task_on_cpu (p_rq, p) || !task_is_running (p)) +return 0 ; + + yielded = curr->sched_class->yield_to_task (rq, p); +if (yielded) { +schedstat_inc (rq->yld_count); +/* +* Make p's CPU reschedule; pick_next_entity +* takes care of fairness. +*/ +if (preempt && rq != p_rq) +resched_curr (p_rq); + } + } -if (yielded >0 ) +if (yielded)schedule ();return yielded;
Peter Zijlstra
maintainerPATCH 5/8 sched: Simplify sched_rr_get_interval()
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-)
kernel/sched/core.c +16 -24
@@ -8941,38 +8941,30 @@ SYSCALL_DEFINE1(sched_get_priority_min,static int sched_rr_get_interval (pid_t pid,struct timespec64 *t) { -struct task_struct *p; -unsigned int time_slice; -struct rq_flags rf; -struct rq *rq; +unsigned int time_slice =0 ;int retval;if (pid <0 )return -EINVAL ; - retval = -ESRCH ; -rcu_read_lock (); - p =find_process_by_pid (pid); -if (!p) - goto out_unlock; - - retval =security_task_getscheduler (p); -if (retval) - goto out_unlock; - - rq =task_rq_lock (p, &rf); - time_slice =0 ; -if (p->sched_class->get_rr_interval) - time_slice = p->sched_class->get_rr_interval (rq, p); -task_rq_unlock (rq, p, &rf); +scoped_guard (rcu) { +struct task_struct *p =find_process_by_pid (pid); +if (!p) +return -ESRCH ; + + retval =security_task_getscheduler (p); +if (retval) +return retval; + + scoped_guard (task_rq_lock, p) { +struct rq *rq = scope.rq; +if (p->sched_class->get_rr_interval) + time_slice = p->sched_class->get_rr_interval (rq, p); + } + } -rcu_read_unlock ();jiffies_to_timespec64 (time_slice, t);return 0 ; - -out_unlock: -rcu_read_unlock (); -return retval; }/**
Peter Zijlstra
maintainerPATCH 6/8 sched: Simplify sched_move_task()
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- kernel/sched/core.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)
kernel/sched/core.c +4 -6
@@ -10361,17 +10361,18 @@ void sched_move_task(struct task_structint queued, running, queue_flags =DEQUEUE_SAVE |DEQUEUE_MOVE |DEQUEUE_NOCLOCK ;struct task_group *group; -struct rq_flags rf;struct rq *rq; - rq =task_rq_lock (tsk, &rf); +CLASS (task_rq_lock, rq_guard)(tsk); + rq = rq_guard.rq; +/* * Esp. with SCHED_AUTOGROUP enabled it is possible to get superfluous * group changes. */ group =sched_get_task_group (tsk);if (group == tsk->sched_task_group) - goto unlock; +return ;update_rq_clock (rq);
@@ -10396,9 +10397,6 @@ void sched_move_task(struct task_struct*/ resched_curr (rq); } - -unlock: -task_rq_unlock (rq, tsk, &rf); }static inline struct task_group *css_tg (struct cgroup_subsys_state *css)
Peter Zijlstra
maintainerPATCH 7/8 sched: Simplify tg_set_cfs_bandwidth()
In reply to Peter ZijlstraUse guards to reduce gotos and simplify control flow.
--- include/linux/cpu.h | 2 ++ kernel/sched/core.c | 42 +++++++++++++++++++++--------------------- 2 files changed, 23 insertions(+), 21 deletions(-)
include/linux/cpu.h +2 -0
@@ -148,6 +148,8 @@ static inline int remove_cpu(unsigned instatic inline void smp_shutdown_nonboot_cpus (unsigned int primary_cpu) { }#endif /* !CONFIG_HOTPLUG_CPU */ +DEFINE_LOCK_GUARD_0(cpus_read_lock, cpus_read_lock(), cpus_read_unlock()) +#ifdef CONFIG_PM_SLEEP_SMP extern int freeze_secondary_cpus (int primary);extern void thaw_secondary_cpus (void );
kernel/sched/core.c +21 -21
71 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 8/8 sched: Misc cleanups
In reply to Peter ZijlstraRandom remaining guard use...
--- kernel/sched/core.c | 167 +++++++++++++++++++--------------------------------- 1 file changed, 63 insertions(+), 104 deletions(-)
kernel/sched/core.c +63 -104
@@ -1480,16 +1480,12 @@ static void __uclamp_update_util_min_rt_static void uclamp_update_util_min_rt_default (struct task_struct *p) { -struct rq_flags rf; -struct rq *rq; -if (!rt_task (p))return ;/* Protect updates to p->uclamp_* */ - rq =task_rq_lock (p, &rf); +guard (task_rq_lock)(p);__uclamp_update_util_min_rt_default (p); -task_rq_unlock (rq, p, &rf); } static inline struct uclamp_se
@@ -1785,9 +1781,8 @@ static void uclamp_update_root_tg(void)uclamp_se_set (&tg->uclamp_req[UCLAMP_MAX ], sysctl_sched_uclamp_util_max, false); -rcu_read_lock (); +guard (rcu)(); cpu_util_update_eff(&root_task_group.css); - rcu_read_unlock(); }#else static void uclamp_update_root_tg (void ) { }
@@ -1814,10 +1809,9 @@ static void uclamp_sync_util_min_rt_defasmp_mb__after_spinlock ();read_unlock (&tasklist_lock); -rcu_read_lock (); +guard (rcu)();for_each_process_thread (g, p)uclamp_update_util_min_rt_default (p); -rcu_read_unlock (); } static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write,
@@ -2250,20 +2244,13 @@ static __always_inlineint task_state_match (struct task_struct *p,unsigned int state) {#ifdef CONFIG_PREEMPT_RT -int match; -/* * Serialize against current_save_and_set_rtlock_wait_state() and * current_restore_rtlock_saved_state(). */ -raw_spin_lock_irq (&p->pi_lock); - match =__task_state_match (p, state); -raw_spin_unlock_irq (&p->pi_lock); - -return match; -#else -return __task_state_match (p, state); +guard (raw_spinlock_irq)(&p->pi_lock);#endif +return __task_state_match (p, state); }/*
@@ -2417,10 +2404,9 @@ void migrate_disable(void)return ; } -preempt_disable (); +guard (preempt)();this_rq ()->nr_pinned++; p->migration_disabled =1 ; -preempt_enable (); }EXPORT_SYMBOL_GPL (migrate_disable);
@@ -2444,7 +2430,7 @@ void migrate_enable(void)* Ensure stop_task runs either before or after this, and that * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). */ -preempt_disable (); +guard (preempt)();if (p->cpus_ptr != &p->cpus_mask)__set_cpus_allowed_ptr (p, &ac);/*
@@ -2455,7 +2441,6 @@ void migrate_enable(void)barrier (); p->migration_disabled =0 ;this_rq ()->nr_pinned--; -preempt_enable (); }EXPORT_SYMBOL_GPL (migrate_enable);
@@ -3516,13 +3501,11 @@ int migrate_swap(struct task_struct *cur*/ void kick_process (struct task_struct *p) { -int cpu; +guard (preempt)(); +int cpu =task_cpu (p); -preempt_disable (); - cpu =task_cpu (p);if ((cpu !=smp_processor_id ()) &&task_curr (p))smp_send_reschedule (cpu); -preempt_enable (); }EXPORT_SYMBOL_GPL (kick_process);
@@ -6367,8 +6350,9 @@ static void sched_core_balance(struct rqstruct sched_domain *sd;int cpu =cpu_of (rq); -preempt_disable (); -rcu_read_lock (); +guard (preempt)(); +guard (rcu)(); +raw_spin_rq_unlock_irq (rq);for_each_domain (cpu, sd) {if (need_resched ())
@@ -6378,8 +6362,6 @@ static void sched_core_balance(struct rqbreak ; }raw_spin_rq_lock_irq (rq); -rcu_read_unlock (); -preempt_enable (); } static DEFINE_PER_CPU(struct balance_callback, core_balance_head);
@@ -8257,8 +8239,6 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pi#ifdef CONFIG_SMP int dl_task_check_affinity (struct task_struct *p,const struct cpumask *mask) { -int ret =0 ; -/* * If the task isn't a deadline task or admission control is * disabled then we don't care about affinity changes.
@@ -8272,11 +8252,11 @@ int dl_task_check_affinity(struct task_s* tasks allowed to run on all the CPUs in the task's * root_domain. */ -rcu_read_lock (); +guard (rcu)();if (!cpumask_subset (task_rq (p)->rd->span, mask)) - ret = -EBUSY ; -rcu_read_unlock (); - return ret; +return -EBUSY ; + + return 0; }#endif
@@ -10508,11 +10488,9 @@ static int cpu_cgroup_css_online(struct#ifdef CONFIG_UCLAMP_TASK_GROUP /* Propagate the effective uclamp value for the new group */ -mutex_lock (&uclamp_mutex); -rcu_read_lock (); +guard (mutex)(&uclamp_mutex); +guard (rcu)();cpu_util_update_eff (css); -rcu_read_unlock (); -mutex_unlock (&uclamp_mutex);#endif return 0 ;
@@ -10663,8 +10641,8 @@ static ssize_t cpu_uclamp_write(struct kstatic_branch_enable (&sched_uclamp_used); -mutex_lock (&uclamp_mutex); -rcu_read_lock (); +guard (mutex)(&uclamp_mutex); +guard (rcu)(); tg =css_tg (of_css (of));if (tg->uclamp_req[clamp_id].value != req.util)
@@ -10679,9 +10657,6 @@ static ssize_t cpu_uclamp_write(struct k/* Update effective clamps to track the most restrictive value */ cpu_util_update_eff (of_css (of)); -rcu_read_unlock (); -mutex_unlock (&uclamp_mutex); -return nbytes; }
@@ -10707,10 +10682,10 @@ static inline void cpu_uclamp_print(struu64 percent;u32 rem; -rcu_read_lock (); - tg =css_tg (seq_css (sf)); - util_clamp = tg->uclamp_req[clamp_id].value; -rcu_read_unlock (); +scoped_guard (rcu) { + tg =css_tg (seq_css (sf)); + util_clamp = tg->uclamp_req[clamp_id].value; + }if (util_clamp ==SCHED_CAPACITY_SCALE ) {seq_puts (sf,"max\n" );
@@ -11032,7 +11007,6 @@ static int tg_cfs_schedulable_down(strucstatic int __cfs_schedulable (struct task_group *tg,u64 period,u64 quota) { -int ret;struct cfs_schedulable_data data = { .tg = tg, .period = period,
@@ -11044,11 +11018,8 @@ static int __cfs_schedulable(struct taskdo_div (data.quota,NSEC_PER_USEC ); } -rcu_read_lock (); - ret =walk_tg_tree (tg_cfs_schedulable_down, tg_nop, &data); -rcu_read_unlock (); - -return ret; +guard (rcu)(); +return walk_tg_tree (tg_cfs_schedulable_down, tg_nop, &data); }static int cpu_cfs_stat_show (struct seq_file *sf,void *v)
@@ -11653,14 +11624,12 @@ int __sched_mm_cid_migrate_from_fetch_ci* are not the last task to be migrated from this cpu for this mm, so * there is no need to move src_cid to the destination cpu. */ -rcu_read_lock (); +guard (rcu)(); src_task =rcu_dereference (src_rq->curr);if (READ_ONCE (src_task->mm_cid_active) && src_task->mm == mm) { -rcu_read_unlock (); t->last_mm_cid =-1 ;return -1 ; } -rcu_read_unlock ();return src_cid; }
@@ -11704,18 +11673,17 @@ int __sched_mm_cid_migrate_from_try_stea* the lazy-put flag, this task will be responsible for transitioning * from lazy-put flag set to MM_CID_UNSET. */ -rcu_read_lock (); - src_task =rcu_dereference (src_rq->curr); -if (READ_ONCE (src_task->mm_cid_active) && src_task->mm == mm) { -rcu_read_unlock (); -/* -* We observed an active task for this mm, there is therefore -* no point in moving this cid to the destination cpu. -*/ - t->last_mm_cid =-1 ; -return -1 ; +scoped_guard (rcu) { + src_task =rcu_dereference (src_rq->curr); +if (READ_ONCE (src_task->mm_cid_active) && src_task->mm == mm) { +/* +* We observed an active task for this mm, there is therefore +* no point in moving this cid to the destination cpu. +*/ + t->last_mm_cid =-1 ; +return -1 ; + } } -rcu_read_unlock ();/* * The src_cid is unused, so it can be unset.
@@ -11788,7 +11756,6 @@ static void sched_mm_cid_remote_clear(st {struct rq *rq =cpu_rq (cpu);struct task_struct *t; -unsigned long flags;int cid, lazy_cid; cid =READ_ONCE (pcpu_cid->cid);
@@ -11823,23 +11790,21 @@ static void sched_mm_cid_remote_clear(st* the lazy-put flag, that task will be responsible for transitioning * from lazy-put flag set to MM_CID_UNSET. */ -rcu_read_lock (); - t =rcu_dereference (rq->curr); -if (READ_ONCE (t->mm_cid_active) && t->mm == mm) { -rcu_read_unlock (); -return ; +scoped_guard (rcu) { + t =rcu_dereference (rq->curr); +if (READ_ONCE (t->mm_cid_active) && t->mm == mm) +return ; } -rcu_read_unlock ();/* * The cid is unused, so it can be unset. * Disable interrupts to keep the window of cid ownership without rq * lock small. */ -local_irq_save (flags); -if (try_cmpxchg (&pcpu_cid->cid, &lazy_cid,MM_CID_UNSET )) -__mm_cid_put (mm, cid); -local_irq_restore (flags); +scoped_guard (irqsave) { +if (try_cmpxchg (&pcpu_cid->cid, &lazy_cid,MM_CID_UNSET )) +__mm_cid_put (mm, cid); + } }static void sched_mm_cid_remote_clear_old (struct mm_struct *mm,int cpu)
@@ -11861,14 +11826,13 @@ static void sched_mm_cid_remote_clear_ol* snapshot associated with this cid if an active task using the mm is * observed on this rq. */ -rcu_read_lock (); - curr =rcu_dereference (rq->curr); -if (READ_ONCE (curr->mm_cid_active) && curr->mm == mm) { -WRITE_ONCE (pcpu_cid->time, rq_clock); -rcu_read_unlock (); -return ; +scoped_guard (rcu) { + curr =rcu_dereference (rq->curr); +if (READ_ONCE (curr->mm_cid_active) && curr->mm == mm) { +WRITE_ONCE (pcpu_cid->time, rq_clock); +return ; + } } -rcu_read_unlock ();if (rq_clock < pcpu_cid->time +SCHED_MM_CID_PERIOD_NS )return ;
@@ -11962,7 +11926,6 @@ void task_tick_mm_cid(struct rq *rq, strvoid sched_mm_cid_exit_signals (struct task_struct *t) {struct mm_struct *mm = t->mm; -struct rq_flags rf;struct rq *rq;if (!mm)
@@ -11970,7 +11933,7 @@ void sched_mm_cid_exit_signals(struct tapreempt_disable (); rq =this_rq (); -rq_lock_irqsave (rq, &rf); +guard (rq_lock_irqsave)(rq);preempt_enable_no_resched ();/* holding spinlock */ WRITE_ONCE (t->mm_cid_active,0 );/*
@@ -11980,13 +11943,11 @@ void sched_mm_cid_exit_signals(struct tasmp_mb ();mm_cid_put (mm); t->last_mm_cid = t->mm_cid =-1 ; -rq_unlock_irqrestore (rq, &rf); }void sched_mm_cid_before_execve (struct task_struct *t) {struct mm_struct *mm = t->mm; -struct rq_flags rf;struct rq *rq;if (!mm)
@@ -11994,7 +11955,7 @@ void sched_mm_cid_before_execve(struct tpreempt_disable (); rq =this_rq (); -rq_lock_irqsave (rq, &rf); +guard (rq_lock_irqsave)(rq);preempt_enable_no_resched ();/* holding spinlock */ WRITE_ONCE (t->mm_cid_active,0 );/*
@@ -12004,13 +11965,11 @@ void sched_mm_cid_before_execve(struct tsmp_mb ();mm_cid_put (mm); t->last_mm_cid = t->mm_cid =-1 ; -rq_unlock_irqrestore (rq, &rf); }void sched_mm_cid_after_execve (struct task_struct *t) {struct mm_struct *mm = t->mm; -struct rq_flags rf;struct rq *rq;if (!mm)
@@ -12018,16 +11977,16 @@ void sched_mm_cid_after_execve(struct tapreempt_disable (); rq =this_rq (); -rq_lock_irqsave (rq, &rf); -preempt_enable_no_resched ();/* holding spinlock */ -WRITE_ONCE (t->mm_cid_active,1 ); -/* -* Store t->mm_cid_active before loading per-mm/cpu cid. -* Matches barrier in sched_mm_cid_remote_clear_old(). -*/ -smp_mb (); - t->last_mm_cid = t->mm_cid =mm_cid_get (rq, mm); -rq_unlock_irqrestore (rq, &rf); + scoped_guard (rq_lock_irqsave, rq) { +preempt_enable_no_resched ();/* holding spinlock */ +WRITE_ONCE (t->mm_cid_active,1 ); +/* +* Store t->mm_cid_active before loading per-mm/cpu cid. +* Matches barrier in sched_mm_cid_remote_clear_old(). +*/ +smp_mb (); + t->last_mm_cid = t->mm_cid =mm_cid_get (rq, mm); + }rseq_set_notify_resume (t); }