sched/rt: optimize checking group rt scheduler constraints
PATCH 1/1 sched/rt: optimize checking group rt scheduler constraints
Group RT scheduler contains protection against setting zero runtime for cgroup with rt tasks. Right now function tg_set_rt_bandwidth() iterates over all cpu cgroups and calls tg_has_rt_tasks() for any cgroup which runtime is zero (not only for changed one). Default rt runtime is zero, thus tg_has_rt_tasks() will is called for almost at cpu cgroups.
This protection already is slightly racy: runtime limit could be changed between cpu_cgroup_can_attach() and cpu_cgroup_attach() because changing cgroup attribute does not lock cgroup_mutex while attach does not lock rt_constraints_mutex. Changing task scheduler class also races with changing rt runtime: check in __sched_setscheduler() isn't protected.
Function tg_has_rt_tasks() iterates over all threads in the system. This gives NR_CGROUPS * NR_TASKS operations under single tasklist_lock locked for read tg_set_rt_bandwidth(). Any concurrent attempt of locking tasklist_lock for write (for example fork) will stuck with disabled irqs.
This patch makes two optimizations:
1) Remove locking tasklist_lock and iterate only tasks in cgroup
2) Call tg_has_rt_tasks() iff rt runtime changes from non-zero to zero
All changed code is under CONFIG_RT_GROUP_SCHED.
Testcase:
# mkdir /sys/fs/cgroup/cpu/test{1..10000}
# echo 0 | tee /sys/fs/cgroup/cpu/test*/cpu.rt_runtime_usAt the same time without patch fork time will be >100ms:
# perf trace -e clone --duration 100 stress-ng --fork 1
Also remote ping will show timings >100ms caused by irq latency.
--- kernel/sched/rt.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)
kernel/sched/rt.c +11 -13
@@ -2396,10 +2396,11 @@ const struct sched_class rt_sched_class = {*/ static DEFINE_MUTEX (rt_constraints_mutex); -/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create().
@@ -2407,12 +2408,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ; -for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task); +css_task_iter_end (&it); -return 0 ; +return ret; }struct rt_schedulable_data {
@@ -2443,9 +2444,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)return -EINVAL ;/* -* Ensure we don't starve existing RT tasks. +* Ensure we don't starve existing RT tasks if runtime turns zero. */ -if (rt_bandwidth_enabled () && !runtime &&tg_has_rt_tasks (tg)) +if (rt_bandwidth_enabled () && !runtime && + tg->rt_bandwidth.rt_runtime &&tg_has_rt_tasks (tg))return -EBUSY ; total =to_ratio (period, runtime);
@@ -2511,7 +2513,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,return -EINVAL ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); err =__rt_schedulable (tg, rt_period, rt_runtime);if (err) goto unlock;
@@ -2529,7 +2530,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, }raw_spin_unlock_irq (&tg->rt_bandwidth.rt_runtime_lock); unlock: -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return err;
@@ -2588,9 +2588,7 @@ static int sched_rt_global_constraints(void)int ret =0 ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); ret =__rt_schedulable (NULL ,0 ,0 ); -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return ret;
On Sat, Jan 25, 2020 at 05:50:38PM +0300, Konstantin Khlebnikov wrote:
...
Quoted 11 lines
kernel/sched/rt.c +3
-2-/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create().
kernel/sched/rt.c static inline int tg_has_rt_tasks(struct task_group *tg) · line 2413
Quoted 10 lines
kernel/sched/rt.c +3
-4@@ -2407,12 +2408,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ; -for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task);
Plain 'ret = rt_task(task);' won't work?
Quoted 5 lines
kernel/sched/rt.c +2
-1+css_task_iter_end (&it); -return 0 ; +return ret; }
Quoted 16 lines · Cyrill Gorcunov
On Sat, Jan 25, 2020 at 05:50:38PM +0300, Konstantin Khlebnikov wrote:
...Quoted 14 lines
kernel/sched/rt.c +3
-2-/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create(). @@ -2407,12 +2408,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ;
Quoted 9 lines
kernel/sched/rt.c static inline int tg_has_rt_tasks(struct task_group *tg) · line 2413
Quoted 7 lines
kernel/sched/rt.c +3
-4-for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task);Plain 'ret =
rt_task(task);' won't work?
Should work too =)
Quoted 6 lines
Quoted 6 lines
kernel/sched/rt.c +2
-1+css_task_iter_end (&it); -return 0 ; +return ret; }
Quoted 7 lines · Konstantin Khlebnikov
Quoted 5 lines
kernel/sched/rt.c static inline int tg_has_rt_tasks(struct task_group *tg) · line 2413
Quoted 3 lines
kernel/sched/rt.c +3
-0+css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task);Plain 'ret =
rt_task(task);' won't work?Should work too =)
:-) no need for new patch then, we could update it on top if needed but i guess compiler would optimize it as is.
Hi Konstantin,
Quoted 58 lines · On Sat, Jan 25, 2020 at 05:50:38PM +0300 Konstantin Khlebnikov wrote:
Group RT scheduler contains protection against setting zero runtime for cgroup with rt tasks. Right now function
tg_set_rt_bandwidth()iterates over all cpu cgroups and callstg_has_rt_tasks()for any cgroup which runtime is zero (not only for changed one). Default rt runtime is zero, thustg_has_rt_tasks()will is called for almost at cpu cgroups.This protection already is slightly racy: runtime limit could be changed between
cpu_cgroup_can_attach()andcpu_cgroup_attach()because changing cgroup attribute does not lockcgroup_mutexwhile attach does not lockrt_constraints_mutex. Changing task scheduler class also races with changing rt runtime: check in__sched_setscheduler()isn't protected.Function
tg_has_rt_tasks()iterates over all threads in the system. This givesNR_CGROUPS*NR_TASKSoperations under singletasklist_locklocked for readtg_set_rt_bandwidth(). Any concurrent attempt of lockingtasklist_lockfor write (for example fork) will stuck with disabled irqs.This patch makes two optimizations:
1) Remove lockingtasklist_lockand iterate only tasks in cgroup
2) Calltg_has_rt_tasks()iff rt runtime changes from non-zero to zeroAll changed code is under
CONFIG_RT_GROUP_SCHED.Testcase:
# mkdir /sys/fs/cgroup/cpu/test{1..10000} # echo 0 | tee /sys/fs/cgroup/cpu/test*/cpu.rt_runtime_usAt the same time without patch fork time will be >100ms:
# perf trace -e clone --duration 100 stress-ng --fork 1Also remote ping will show timings >100ms caused by irq latency.
Signed-off-by Konstantin Khlebnikov <[email protected]>--- kernel/sched/rt.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)kernel/sched/rt.c +3
-2@@ -2396,10 +2396,11 @@ const struct sched_class rt_sched_class = {*/ static DEFINE_MUTEX (rt_constraints_mutex); -/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create(). @@ -2407,12 +2408,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ;
kernel/sched/rt.c static inline int tg_has_rt_tasks(struct task_group *tg) · line 2415
Quoted 10 lines
kernel/sched/rt.c +4
-4-for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task); +css_task_iter_end (&it);
Why not a break; in there alon with the change to "="?
Quoted 36 lines
kernel/sched/rt.c +4
-5-return 0 ; +return ret; }struct rt_schedulable_data {@@ -2443,9 +2444,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)return -EINVAL ;/* -* Ensure we don't starve existing RT tasks. +* Ensure we don't starve existing RT tasks if runtime turns zero. */ -if (rt_bandwidth_enabled () && !runtime &&tg_has_rt_tasks (tg)) +if (rt_bandwidth_enabled () && !runtime && + tg->rt_bandwidth.rt_runtime &&tg_has_rt_tasks (tg))return -EBUSY ; total =to_ratio (period, runtime);@@ -2511,7 +2513,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,return -EINVAL ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); err =__rt_schedulable (tg, rt_period, rt_runtime);if (err) goto unlock;@@ -2529,7 +2530,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, }raw_spin_unlock_irq (&tg->rt_bandwidth.rt_runtime_lock); unlock: -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return err;
kernel/sched/rt.c static int sched_rt_global_constraints(void) · line 2594
Quoted 9 lines
kernel/sched/rt.c +0
-2@@ -2588,9 +2588,7 @@ static int sched_rt_global_constraints(void)int ret =0 ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); ret =__rt_schedulable (NULL ,0 ,0 ); -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return ret;
Thanks,
Phil
[tip: sched/core] sched/rt: Optimize checking group RT scheduler constraints
In reply to Konstantin KhlebnikovThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: b4fb015eeff7f3e5518a7dbe8061169a3e2f2bc7 Gitweb: https://git.kernel.org/tip/b4fb015eeff7f3e5518a7dbe8061169a3e2f2bc7 Author: Konstantin Khlebnikov <[email protected]> AuthorDate: Sat, 25 Jan 2020 17:50:38 +03:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 28 Jan 2020 21:37:09 +01:00
sched/rt: Optimize checking group RT scheduler constraints
Group RT scheduler contains protection against setting zero runtime for cgroup with RT tasks. Right now function tg_set_rt_bandwidth() iterates over all CPU cgroups and calls tg_has_rt_tasks() for any cgroup which runtime is zero (not only for changed one). Default RT runtime is zero, thus tg_has_rt_tasks() will is called for almost at CPU cgroups.
This protection already is slightly racy: runtime limit could be changed between cpu_cgroup_can_attach() and cpu_cgroup_attach() because changing cgroup attribute does not lock cgroup_mutex while attach does not lock rt_constraints_mutex. Changing task scheduler class also races with changing rt runtime: check in __sched_setscheduler() isn't protected.
Function tg_has_rt_tasks() iterates over all threads in the system. This gives NR_CGROUPS * NR_TASKS operations under single tasklist_lock locked for read tg_set_rt_bandwidth(). Any concurrent attempt of locking tasklist_lock for write (for example fork) will stuck with disabled irqs.
This patch makes two optimizations:
1) Remove locking tasklist_lock and iterate only tasks in cgroup
2) Call tg_has_rt_tasks() iff rt runtime changes from non-zero to zero
All changed code is under CONFIG_RT_GROUP_SCHED.
Testcase:
# mkdir /sys/fs/cgroup/cpu/test{1..10000}
# echo 0 | tee /sys/fs/cgroup/cpu/test*/cpu.rt_runtime_usAt the same time without patch fork time will be >100ms:
# perf trace -e clone --duration 100 stress-ng --fork 1
Also remote ping will show timings >100ms caused by irq latency.
--- kernel/sched/rt.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)
kernel/sched/rt.c +11 -13
@@ -2449,10 +2449,11 @@ const struct sched_class rt_sched_class = {*/ static DEFINE_MUTEX (rt_constraints_mutex); -/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create().
@@ -2460,12 +2461,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ; -for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task); +css_task_iter_end (&it); -return 0 ; +return ret; }struct rt_schedulable_data {
@@ -2496,9 +2497,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)return -EINVAL ;/* -* Ensure we don't starve existing RT tasks. +* Ensure we don't starve existing RT tasks if runtime turns zero. */ -if (rt_bandwidth_enabled () && !runtime &&tg_has_rt_tasks (tg)) +if (rt_bandwidth_enabled () && !runtime && + tg->rt_bandwidth.rt_runtime &&tg_has_rt_tasks (tg))return -EBUSY ; total =to_ratio (period, runtime);
@@ -2564,7 +2566,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,return -EINVAL ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); err =__rt_schedulable (tg, rt_period, rt_runtime);if (err) goto unlock;
@@ -2582,7 +2583,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, }raw_spin_unlock_irq (&tg->rt_bandwidth.rt_runtime_lock); unlock: -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return err;
@@ -2641,9 +2641,7 @@ static int sched_rt_global_constraints(void)int ret =0 ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); ret =__rt_schedulable (NULL ,0 ,0 ); -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return ret;
Re: [tip: sched/core] sched/rt: Optimize checking group RT scheduler constraints
Quoted 74 lines · On Wed, Jan 29, 2020 at 11:32:56AM -0000 tip-bot2 for Konstantin Khlebnikov…
The following commit has been merged into the sched/core branch of tip:
Commit-ID: b4fb015eeff7f3e5518a7dbe8061169a3e2f2bc7 Gitweb: https://git.kernel.org/tip/b4fb015eeff7f3e5518a7dbe8061169a3e2f2bc7 Author: Konstantin Khlebnikov <[email protected]> AuthorDate: Sat, 25 Jan 2020 17:50:38 +03:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Tue, 28 Jan 2020 21:37:09 +01:00sched/rt: Optimize checking group RT scheduler constraints
Group RT scheduler contains protection against setting zero runtime for cgroup with RT tasks. Right now function
tg_set_rt_bandwidth()iterates over all CPU cgroups and callstg_has_rt_tasks()for any cgroup which runtime is zero (not only for changed one). Default RT runtime is zero, thustg_has_rt_tasks()will is called for almost at CPU cgroups.This protection already is slightly racy: runtime limit could be changed between
cpu_cgroup_can_attach()andcpu_cgroup_attach()because changing cgroup attribute does not lockcgroup_mutexwhile attach does not lockrt_constraints_mutex. Changing task scheduler class also races with changing rt runtime: check in__sched_setscheduler()isn't protected.Function
tg_has_rt_tasks()iterates over all threads in the system. This givesNR_CGROUPS*NR_TASKSoperations under singletasklist_locklocked for readtg_set_rt_bandwidth(). Any concurrent attempt of lockingtasklist_lockfor write (for example fork) will stuck with disabled irqs.This patch makes two optimizations:
1) Remove lockingtasklist_lockand iterate only tasks in cgroup
2) Calltg_has_rt_tasks()iff rt runtime changes from non-zero to zeroAll changed code is under
CONFIG_RT_GROUP_SCHED.Testcase:
# mkdir /sys/fs/cgroup/cpu/test{1..10000} # echo 0 | tee /sys/fs/cgroup/cpu/test*/cpu.rt_runtime_usAt the same time without patch fork time will be >100ms:
# perf trace -e clone --duration 100 stress-ng --fork 1Also remote ping will show timings >100ms caused by irq latency.
Signed-off-by Konstantin Khlebnikov <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Signed-off-by Ingo Molnar <[email protected]>--- kernel/sched/rt.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-)kernel/sched/rt.c +3
-2@@ -2449,10 +2449,11 @@ const struct sched_class rt_sched_class = {*/ static DEFINE_MUTEX (rt_constraints_mutex); -/* Must be called with tasklist_lock held */ static inline int tg_has_rt_tasks (struct task_group *tg) { -struct task_struct *g, *p; +struct task_struct *task; +struct css_task_iter it; +int ret =0 ;/* * Autogroups do not have RT tasks; see autogroup_create(). @@ -2460,12 +2461,12 @@ static inline int tg_has_rt_tasks(struct task_group *tg)if (task_group_is_autogroup (tg))return 0 ;
kernel/sched/rt.c static inline int tg_has_rt_tasks(struct task_group *tg) · line 2468
Quoted 10 lines
kernel/sched/rt.c +4
-4-for_each_process_thread (g, p) { -if (rt_task (p) &&task_group (p) == tg) -return 1 ; - } +css_task_iter_start (&tg->css,0 , &it); +while (!ret && (task =css_task_iter_next (&it))) + ret |=rt_task (task); +css_task_iter_end (&it);
Heh, I misread it the first time and didn't see the !ret condition. But I think it should be just "ret = ".
But thanks for getting this fixed.
Cheers,
Phil
Quoted 45 lines
kernel/sched/rt.c +4
-7-return 0 ; +return ret; }struct rt_schedulable_data {@@ -2496,9 +2497,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)return -EINVAL ;/* -* Ensure we don't starve existing RT tasks. +* Ensure we don't starve existing RT tasks if runtime turns zero. */ -if (rt_bandwidth_enabled () && !runtime &&tg_has_rt_tasks (tg)) +if (rt_bandwidth_enabled () && !runtime && + tg->rt_bandwidth.rt_runtime &&tg_has_rt_tasks (tg))return -EBUSY ; total =to_ratio (period, runtime);@@ -2564,7 +2566,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg,return -EINVAL ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); err =__rt_schedulable (tg, rt_period, rt_runtime);if (err) goto unlock;@@ -2582,7 +2583,6 @@ static int tg_set_rt_bandwidth(struct task_group *tg, }raw_spin_unlock_irq (&tg->rt_bandwidth.rt_runtime_lock); unlock: -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return err;@@ -2641,9 +2641,7 @@ static int sched_rt_global_constraints(void)int ret =0 ;mutex_lock (&rt_constraints_mutex); -read_lock (&tasklist_lock); ret =__rt_schedulable (NULL ,0 ,0 ); -read_unlock (&tasklist_lock);mutex_unlock (&rt_constraints_mutex);return ret;