sched: rt-group: smp balancing
Subsystems (5)
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/8 sched: fix incorrect irq lock usage in normalize_rt_tasks()
lockdep spotted this bogus irq locking. normalize_rt_tasks() can be called from hardirq context through sysrq-n
--- kernel/sched.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
kernel/sched.c +4 -4
@@ -7345,7 +7345,7 @@ void normalize_rt_tasks(void)unsigned long flags;struct rq *rq; -read_lock_irq (&tasklist_lock); +read_lock_irqsave (&tasklist_lock, flags); do_each_thread(g, p) {/* * Only normalize user tasks:
@@ -7371,16 +7371,16 @@ void normalize_rt_tasks(void) continue; } -spin_lock_irqsave (&p->pi_lock, flags); +spin_lock (&p->pi_lock); rq =__task_rq_lock (p);normalize_task (rq, p);__task_rq_unlock (rq); -spin_unlock_irqrestore (&p->pi_lock, flags); +spin_unlock (&p->pi_lock); }while_each_thread (g, p); -read_unlock_irq (&tasklist_lock); +read_unlock_irqrestore (&tasklist_lock, flags); }#endif /* CONFIG_MAGIC_SYSRQ */
--
PATCH 2/8 sched: rt-group: deal with PI
In reply to a message not in the archiveSteven mentioned the fun case where a lock holding task will be throttled.
Simple fix: allow groups that have boosted tasks to run anyway.
If a runnable task in a throttled group gets boosted the dequeue/enqueue done by rt_mutex_setprio() is enough to unthrottle the group.
This is ofcourse not quite correct. Two possible ways forward are:
- second prio array for boosted tasks
- boost to a prio ceiling (this would also work for deadline scheduling)
--- kernel/sched.c | 3 +++ kernel/sched_rt.c | 43 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-)
kernel/sched.c +3 -0
@@ -374,6 +374,8 @@ struct rt_rq {u64 rt_time;#ifdef CONFIG_FAIR_GROUP_SCHED +unsigned long rt_nr_boosted; +struct rq *rq;struct list_head leaf_rt_rq_list;struct task_group *tg;
@@ -7116,6 +7118,7 @@ static void init_rt_rq(struct rt_rq *rt_ rt_rq->rt_throttled =0 ;#ifdef CONFIG_FAIR_GROUP_SCHED + rt_rq->rt_nr_boosted =0 ; rt_rq->rq = rq;#endif }
kernel/sched_rt.c +38 -5
91 lines not on this page · raw message
--
PATCH 3/8 sched: rt-group: interface
In reply to a message not in the archiveChange the rt_ratio interface to rt_runtime_us, to match rt_period_us. This avoids picking a granularity for the ratio.
Extend the /sys/kernel/uids/<uid>/ interface to allow setting the group's rt_runtime.
--- Documentation/ABI/testing/sysfs-kernel-uids | 6 + Documentation/sched-rt-group.txt | 59 +++++++++++ include/linux/sched.h | 7 - kernel/sched.c | 145 +++++++++++++++++++++------- kernel/sched_rt.c | 53 ++++------ kernel/sysctl.c | 32 +++--- kernel/user.c | 28 +++++ 7 files changed, 250 insertions(+), 80 deletions(-)
include/linux/sched.h +5 -2
@@ -1507,8 +1507,6 @@ extern unsigned int sysctl_sched_child_rextern unsigned int sysctl_sched_features;extern unsigned int sysctl_sched_migration_cost;extern unsigned int sysctl_sched_nr_migrate; -extern unsigned int sysctl_sched_rt_period; -extern unsigned int sysctl_sched_rt_ratio;#if defined(CONFIG_FAIR_GROUP_SCHED ) && defined(CONFIG_SMP )extern unsigned int sysctl_sched_min_bal_int_shares;extern unsigned int sysctl_sched_max_bal_int_shares;
@@ -1518,6 +1516,8 @@ int sched_nr_latency_handler(struct ctl_ struct file *file, void __user *buffer, size_t *length, loff_t *ppos);#endif +extern unsigned int sysctl_sched_rt_period; +extern int sysctl_sched_rt_runtime;extern unsigned int sysctl_sched_compat_yield;
@@ -1997,6 +1997,9 @@ extern void sched_destroy_group(struct textern void sched_move_task (struct task_struct *tsk);extern int sched_group_set_shares (struct task_group *tg,unsigned long shares);extern unsigned long sched_group_shares (struct task_group *tg); +extern int sched_group_set_rt_runtime (struct task_group *tg, +long rt_runtime_us); +extern long sched_group_rt_runtime (struct task_group *tg);#endif
kernel/sched.c +113 -32
209 lines not on this page · raw message
kernel/sched_rt.c +23 -30
126 lines not on this page · raw message
kernel/sysctl.c +16 -16
@@ -311,22 +311,6 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, - { - .ctl_name = CTL_UNNUMBERED, - .procname = "sched_rt_period_ms", - .data = &sysctl_sched_rt_period, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = &proc_dointvec, - }, - { - .ctl_name = CTL_UNNUMBERED, - .procname = "sched_rt_ratio", - .data = &sysctl_sched_rt_ratio, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = &proc_dointvec, - },#if defined(CONFIG_FAIR_GROUP_SCHED ) && defined(CONFIG_SMP ) { .ctl_name = CTL_UNNUMBERED,
@@ -348,6 +332,22 @@ static struct ctl_table kern_table[] = {#endif { .ctl_name = CTL_UNNUMBERED, + .procname = "sched_rt_period_us", + .data = &sysctl_sched_rt_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_rt_runtime_us", + .data = &sysctl_sched_rt_runtime, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec, + }, + { + .ctl_name = CTL_UNNUMBERED, .procname = "sched_compat_yield", .data = &sysctl_sched_compat_yield, .maxlen = sizeof(unsigned int),
Documentation/ABI/testing/sysfs-kernel-uids +6 -0
@@ -12,3 +12,9 @@ Description: B has shares = 2048, User B will get twice the CPU bandwidth user A will. For more details refer Documentation/sched-design-CFS.txt + +What: /sys/kernel/uids/<uid>/cpu_rt_runtime_us +Date: January 2008 +Contact: Peter Zijlstra <[email protected]> +Description: See Documentation/sched-rt-group.txt +
Documentation/sched-rt-group.txt new file +59 -0
59 lines not on this page · raw message
kernel/user.c +28 -0
@@ -156,9 +156,37 @@ static ssize_t cpu_shares_store(struct kstatic struct kobj_attribute cpu_share_attr =__ATTR (cpu_share,0644 , cpu_shares_show, cpu_shares_store); +static ssize_t cpu_rt_runtime_show (struct kobject *kobj, +struct kobj_attribute *attr, +char *buf) +{ + struct user_struct *up = container_of(kobj, struct user_struct, kobj); + +return sprintf (buf,"%lu\n" ,sched_group_rt_runtime (up->tg)); +} + +static ssize_t cpu_rt_runtime_store (struct kobject *kobj, +struct kobj_attribute *attr, +const char *buf,size_t size) +{ + struct user_struct *up = container_of(kobj, struct user_struct, kobj); +unsigned long rt_runtime; +int rc; + +sscanf (buf,"%lu" , &rt_runtime); + + rc =sched_group_set_rt_runtime (up->tg, rt_runtime); + +return (rc ? rc : size); +} + +static struct kobj_attribute cpu_rt_runtime_attr = +__ATTR (cpu_rt_runtime,0644 , cpu_rt_runtime_show, cpu_rt_runtime_store); +/* default attributes per uid directory */ static struct attribute *uids_attributes[] = { &cpu_share_attr.attr, + &cpu_rt_runtime_attr.attr,NULL };
--
Re: [PATCH 3/8] sched: rt-group: interface
Quoted 28 lines · On Mon, 04 Feb 2008 22:03:01 +0100 Peter Zijlstra wrote:
Change the
rt_ratiointerface tort_runtime_us, to matchrt_period_us. This avoids picking a granularity for the ratio.Extend the /sys/kernel/uids/<uid>/ interface to allow setting the group's
rt_runtime.Signed-off-by Peter Zijlstra <[email protected]>--- Documentation/ABI/testing/sysfs-kernel-uids | 6 + Documentation/sched-rt-group.txt | 59 +++++++++++ include/linux/sched.h | 7 - kernel/sched.c | 145 +++++++++++++++++++++------- kernel/sched_rt.c | 53 ++++------ kernel/sysctl.c | 32 +++--- kernel/user.c | 28 +++++ 7 files changed, 250 insertions(+), 80 deletions(-)Index:
linux-2.6/kernel/sched.c
===================================================================kernel/sched.c +1
-1@@ -7780,30 +7783,76 @@ unsigned long sched_group_shares(struct }/* -* Ensure the total rt_ratio <= sysctl_sched_rt_ratio +* Ensure that the real time constraints are schedulable. */
kernel/sched.c unsigned long sched_group_shares(struct · line 7796
Quoted 10 lines
kernel/sched.c +9
-1-int sched_group_set_rt_ratio (struct task_group *tg,unsigned long rt_ratio) +static DEFINE_MUTEX (rt_constraints_mutex); + +static unsigned long to_ratio (u64 period,u64 runtime) +{ +if (runtime ==RUNTIME_INF ) +return 1ULL <<16 ; + + runtime *= (1ULL <<16 ); +do_div (runtime, period);
Isn't do_div() defined as taking (uint64_t, uint32_t) ?
Quoted 9 lines
kernel/sched.c +3
-0+return runtime; +} +Index:
linux-2.6/Documentation/sched-rt-group.txt
===================================================================
Documentation/sched-rt-group.txt line 8
Quoted 8 lines
Documentation/sched-rt-group.txt new file +8
-0@@ -0,0 +1,59 @@ + + +Real-Time group scheduling. + +The problem space: + +In order to schedule multiple groups of realtime tasks each group must +be assigned a fixed portion of the cpu time available. Without a minimum
Use "cpu" or "CPU" consistently, please. (I prefer CPU, but ....)
Documentation/sched-rt-group.txt line 13
Quoted 5 lines
Documentation/sched-rt-group.txt new file +5
-0+guarantee a realtime group can obviously fall short. A fuzzy upper limit +is of no use since it cannot be relied upon. Which leaves us with just +the single fixed portion. + +CPU time is divided by means of specifying how much time can be spend
s/spend/spent/
Documentation/sched-rt-group.txt line 15
Quoted 2 lines
Documentation/sched-rt-group.txt new file +2
-0+running in a given period. Say a frame fixed realtime renderer must +deliver a 25 frames a second, which yields a period of 0.04s. Now say
drop "a"^
Documentation/sched-rt-group.txt line 23
Quoted 8 lines
Documentation/sched-rt-group.txt new file +8
-0+it will also have to play some music and respond to input, leaving it +with around 80% for the graphics. We can then give this group a runtime +of 0.8 * 0.04s = 0.032s. + +This way the graphics group will have a 0.04s period with a 0.032s runtime +limit. + +Now if the audio thread needs to refill the dma buffer every 0.005s, but
DMA preferably.
Documentation/sched-rt-group.txt line 24
Quoted 1 line
Documentation/sched-rt-group.txt new file +1
-0+needs only about 3% cpu time to do so, it will can do with a 0.03 * 0.005s
s/will can do/can do/
Quoted 11 lines
Documentation/sched-rt-group.txt new file +11
-0+= 0.00015s. + + +The Interface: + +system wide: + +/proc/sys/kernel/sched_rt_period_ms +/proc/sys/kernel/sched_rt_runtime_us + +CONFIG_FAIR_USER_SCHED
Documentation/sched-rt-group.txt line 45
Quoted 10 lines
Documentation/sched-rt-group.txt new file +10
-0+ +/sys/kernel/uids/<uid>/cpu_rt_runtime_us + +or + +CONFIG_FAIR_CGROUP_SCHED + +/cgroup/<cgroup>/cpu.rt_runtime_us + +[ time is specified in us because the interface is s32, this gives an
s/,/;/
Documentation/sched-rt-group.txt line 55
Quoted 10 lines
Documentation/sched-rt-group.txt new file +10
-0+ operating range of ~35m to 1us ] + +The period takes values in [ 1, INT_MAX ], runtime in [ -1, INT_MAX - 1 ]. + +A runtime of -1 specifies runtime == period, ie. no limit. + +New groups get the period from /proc/sys/kernel/sched_rt_period_us and +a runtime of 0. + +Settings are constrainted to:
constrained
Documentation/sched-rt-group.txt line 59
Quoted 4 lines
Documentation/sched-rt-group.txt new file +4
-0+ + \Sum_{i} runtime_{i} / global_period <= global_runtime / global_period + +in order to keep the configuration schedulable.
---
~Randy
Re: [PATCH 3/8] sched: rt-group: interface
In reply to Peter ZijlstraQuoted 45 lines · Peter Zijlstra
+static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
+ struct file *file, + const char __user *userbuf, + size_t nbytes, loff_t *unused_ppos) +{ + char buffer[64]; + int retval = 0; + s64 val; + char *end; + + if (!nbytes) + return -EINVAL; + if (nbytes >= sizeof(buffer)) + return -E2BIG; + if (copy_from_user(buffer, userbuf, nbytes)) + return -EFAULT; + + buffer[nbytes] = 0; /* nul-terminate */ + + /* strip newline if necessary */ + if (nbytes && (buffer[nbytes-1] == '\n')) + buffer[nbytes-1] = 0; + val = simple_strtoll(buffer, &end, 0); + if (*end) + return -EINVAL; + + /* Pass to subsystem */ + retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val); + if (!retval)+ retval = nbytes;
+ return retval;
}-static u64
cpu_rt_ratio_read_uint(struct cgroup *cgrp, struct cftype *cft)-{
- structtask_group*tg =cgroup_tg(cgrp);
+staticssize_tcpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,+ struct file *file, + char __user *buf, size_t nbytes, + loff_t *ppos) +{ + char tmp[64]; + long val = sched_group_rt_runtime(cgroup_tg(cgrp)); + int len = sprintf(tmp, "%ld\n", val);
Quoted 3 lines
- return (u64)
tg->rt_ratio;
+ returnsimple_read_from_buffer(buf, nbytes, ppos, tmp, len);
}
What's the reason that you can't use the cgroup read_uint/write_uint methods for this? Is it just because you have -1 as your "unlimited" value.
If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.
Paul
Re: [PATCH 3/8] sched: rt-group: interface
Quoted 46 lines · Paul Menage
Quoted 45 lines · Peter Zijlstra
+static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
+ struct file *file, + const char __user *userbuf, + size_t nbytes, loff_t *unused_ppos) +{ + char buffer[64]; + int retval = 0; + s64 val; + char *end; + + if (!nbytes) + return -EINVAL; + if (nbytes >= sizeof(buffer)) + return -E2BIG; + if (copy_from_user(buffer, userbuf, nbytes)) + return -EFAULT; + + buffer[nbytes] = 0; /* nul-terminate */ + + /* strip newline if necessary */ + if (nbytes && (buffer[nbytes-1] == '\n')) + buffer[nbytes-1] = 0; + val = simple_strtoll(buffer, &end, 0); + if (*end) + return -EINVAL; + + /* Pass to subsystem */ + retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val); + if (!retval)+ retval = nbytes;
+ return retval;
}-static u64
cpu_rt_ratio_read_uint(struct cgroup *cgrp, struct cftype *cft)-{
- structtask_group*tg =cgroup_tg(cgrp);
+staticssize_tcpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,+ struct file *file, + char __user *buf, size_t nbytes, + loff_t *ppos) +{ + char tmp[64]; + long val = sched_group_rt_runtime(cgroup_tg(cgrp)); + int len = sprintf(tmp, "%ld\n", val);
Quoted 7 lines
Quoted 3 lines
- return (u64)
tg->rt_ratio;
+ returnsimple_read_from_buffer(buf, nbytes, ppos, tmp, len);
}What's the reason that you can't use the cgroup read_uint/write_uint methods for this? Is it just because you have -1 as your "unlimited" value.
Yes.
Quoted 3 lines
If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.
0 means no time, quite useful and clearly distinct from inf. time.
Re: [PATCH 3/8] sched: rt-group: interface
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines
If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.
0 means no time, quite useful and clearly distinct from inf. time.
So a real-time task in a cgroup with a 0 rt_runtime can be in the R state but never actually get to run? OK, if people need to be able to do that then fair enough.
In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.
Paul
Re: [PATCH 3/8] sched: rt-group: interface
Quoted 11 lines · Paul Menage
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines
If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.
0 means no time, quite useful and clearly distinct from inf. time.
So a real-time task in a cgroup with a 0
rt_runtimecan be in the R state but never actually get to run? OK, if people need to be able to do that then fair enough.
Yeah, its an awkward situation, and we refuse new rt tasks in such groups. But the 0 value is needed so you can have groups that don't participate in the realtime scheduling because we enforce a schedulalbility constraint over the groups.
Each group has a runtime ratio, namely: rt_runtime / rt_period. The sum of this ratio over all groups must be smaller or equal to the global ratio which must be smaller or equal to 1.
Quoted 2 lines
In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.
Yes, I looked at that, I found the interface somewhat unfortunate, it would mean growing the struct with two more function pointers. Perhaps a read and write function with abstract data would be better suited. That would allow for this and more. Sadly it looses type information.
Re: [PATCH 3/8] sched: rt-group: interface
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines
In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.
Yes, I looked at that, I found the interface somewhat unfortunate, it would mean growing the struct with two more function pointers.
Is that really a big deal? We're talking about a structure that has a small number (<10 in the current tree) of instances per cgroup subsystem.
Quoted 3 lines
Perhaps a
read and write function with abstract data would be better suited. That would allow for this and more. Sadly it looses type information.
If the size of the struct cftype really became a problem, I think the cleanest way to fix it would be to have a union of the potential function pointers, and add a field to specify which one is in use.
Paul
PATCH 4/8 sched: rt-group: make rt groups scheduling configurable
In reply to a message not in the archiveMake the rt group scheduler compile time configurable. Enable it by default for cgroup scheduling.
--- include/linux/cgroup_subsys.h | 2 init/Kconfig | 23 +++++-- kernel/sched.c | 130 +++++++++++++++++++++++++++++++----------- kernel/sched_rt.c | 12 +-- 4 files changed, 120 insertions(+), 47 deletions(-)
include/linux/cgroup_subsys.h +1 -1
@@ -25,7 +25,7 @@ SUBSYS(ns)/* */ -#ifdef CONFIG_FAIR_CGROUP_SCHED +#ifdef CONFIG_CGROUP_SCHED SUBSYS(cpu_cgroup)#endif
init/Kconfig +17 -6
@@ -332,25 +332,36 @@ config CPUSETS Say N if unsure. -config FAIR_GROUP_SCHED - bool "Fair group CPU scheduler" +config GROUP_SCHED + bool "Group CPU scheduler" default y help This feature lets CPU scheduler recognize task groups and control CPU bandwidth allocation to such task groups. +config FAIR_GROUP_SCHED + bool "Group scheduling for SCHED_OTHER" + depends on GROUP_SCHED + default y + +config RT_GROUP_SCHED + bool "Group scheduling for SCHED_RR/FIFO" + depends on EXPERIMENTAL + depends on GROUP_SCHED + default n + choice - depends on FAIR_GROUP_SCHED + depends on GROUP_SCHED prompt "Basis for grouping tasks" - default FAIR_USER_SCHED + default USER_SCHED -config FAIR_USER_SCHED +config USER_SCHED bool "user id" help This option will choose userid as the basis for grouping tasks, thus providing equal CPU bandwidth to each user. -config FAIR_CGROUP_SCHED +config CGROUP_SCHED bool "Control groups" depends on CGROUPS help
kernel/sched.c +96 -34
420 lines not on this page · raw message
kernel/sched_rt.c +6 -6
@@ -55,7 +55,7 @@ static inline int on_rt_rq(struct sched_return !list_empty (&rt_se->run_list); } -#ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_RT_GROUP_SCHED static inline u64 sched_rt_runtime (struct rt_rq *rt_rq) {
@@ -177,7 +177,7 @@ static inline int rt_rq_throttled(structstatic inline int rt_se_prio (struct sched_rt_entity *rt_se) { -#ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_RT_GROUP_SCHED struct rt_rq *rt_rq =group_rt_rq (rt_se);if (rt_rq)
@@ -269,7 +269,7 @@ void inc_rt_tasks(struct sched_rt_entity {WARN_ON (!rt_prio (rt_se_prio (rt_se))); rt_rq->rt_nr_running++; -#if definedCONFIG_SMP || definedCONFIG_FAIR_GROUP_SCHED +#if definedCONFIG_SMP || definedCONFIG_RT_GROUP_SCHED if (rt_se_prio (rt_se) < rt_rq->highest_prio) rt_rq->highest_prio =rt_se_prio (rt_se);#endif
@@ -281,7 +281,7 @@ void inc_rt_tasks(struct sched_rt_entityupdate_rt_migration (rq_of_rt_rq (rt_rq));#endif -#ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted (rt_se)) rt_rq->rt_nr_boosted++;#endif
@@ -293,7 +293,7 @@ void dec_rt_tasks(struct sched_rt_entityWARN_ON (!rt_prio (rt_se_prio (rt_se)));WARN_ON (!rt_rq->rt_nr_running); rt_rq->rt_nr_running--; -#if definedCONFIG_SMP || definedCONFIG_FAIR_GROUP_SCHED +#if definedCONFIG_SMP || definedCONFIG_RT_GROUP_SCHED if (rt_rq->rt_nr_running) {struct rt_prio_array *array;
@@ -315,7 +315,7 @@ void dec_rt_tasks(struct sched_rt_entityupdate_rt_migration (rq_of_rt_rq (rt_rq));#endif /* CONFIG_SMP */ -#ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted (rt_se)) rt_rq->rt_nr_boosted--;
--
PATCH 5/8 sched: rt-group: clean up the ifdeffery
In reply to a message not in the archiveClean up some of the excessive ifdeffery introduces in the last patch.
--- kernel/sched.c | 150 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 50 deletions(-)
kernel/sched.c +100 -50
@@ -7563,56 +7563,29 @@ static int load_balance_monitor(void *un }#endif /* CONFIG_SMP */ -static void free_sched_group (struct task_group *tg) +#ifdef CONFIG_FAIR_GROUP_SCHED +static void free_fair_sched_group (struct task_group *tg) {int i;for_each_possible_cpu (i) { -#ifdef CONFIG_FAIR_GROUP_SCHED if (tg->cfs_rq)kfree (tg->cfs_rq[i]);if (tg->se)kfree (tg->se[i]); -#endif -#ifdef CONFIG_RT_GROUP_SCHED -if (tg->rt_rq) -kfree (tg->rt_rq[i]); -if (tg->rt_se) -kfree (tg->rt_se[i]); -#endif } -#ifdef CONFIG_FAIR_GROUP_SCHED kfree (tg->cfs_rq);kfree (tg->se); -#endif -#ifdef CONFIG_RT_GROUP_SCHED -kfree (tg->rt_rq); -kfree (tg->rt_se); -#endif -kfree (tg); } -/* allocate runqueue etc for a new task group */ -struct task_group *sched_create_group (void ) +static int alloc_fair_sched_group (struct task_group *tg) { -struct task_group *tg; -#ifdef CONFIG_FAIR_GROUP_SCHED struct cfs_rq *cfs_rq;struct sched_entity *se; -#endif -#ifdef CONFIG_RT_GROUP_SCHED -struct rt_rq *rt_rq; -struct sched_rt_entity *rt_se; -#endif struct rq *rq;int i; - tg =kzalloc (sizeof (*tg),GFP_KERNEL ); -if (!tg) -return ERR_PTR (-ENOMEM ); - -#ifdef CONFIG_FAIR_GROUP_SCHED tg->cfs_rq =kzalloc (sizeof (cfs_rq) *NR_CPUS ,GFP_KERNEL );if (!tg->cfs_rq) goto err;
@@ -7621,23 +7594,10 @@ struct task_group *sched_create_group(vo goto err; tg->shares =NICE_0_LOAD ; -#endif - -#ifdef CONFIG_RT_GROUP_SCHED - tg->rt_rq =kzalloc (sizeof (rt_rq) *NR_CPUS ,GFP_KERNEL ); -if (!tg->rt_rq) - goto err; - tg->rt_se =kzalloc (sizeof (rt_se) *NR_CPUS ,GFP_KERNEL ); -if (!tg->rt_se) - goto err; - - tg->rt_runtime =0 ; -#endif for_each_possible_cpu (i) { rq =cpu_rq (i); -#ifdef CONFIG_FAIR_GROUP_SCHED cfs_rq =kmalloc_node (sizeof (struct cfs_rq ),GFP_KERNEL |__GFP_ZERO,cpu_to_node (i));if (!cfs_rq)
@@ -7649,9 +7609,68 @@ struct task_group *sched_create_group(vo goto err; init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0); + } + +lock_task_group_list (); +for_each_possible_cpu (i) { + rq =cpu_rq (i); + cfs_rq = tg->cfs_rq[i]; +list_add_rcu (&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); + } +list_add_rcu (&tg->list, &task_groups); +unlock_task_group_list (); + +return 1 ; + + err: +return 0 ; +} +#else +static inline void free_fair_sched_group (struct task_group *tg) +{ +} + +static inline int alloc_fair_sched_group (struct task_group *tg) +{ +return 1 ; +}#endif #ifdef CONFIG_RT_GROUP_SCHED +static void free_rt_sched_group (struct task_group *tg) +{ +int i; + +for_each_possible_cpu (i) { +if (tg->rt_rq) +kfree (tg->rt_rq[i]); +if (tg->rt_se) +kfree (tg->rt_se[i]); + } + +kfree (tg->rt_rq); +kfree (tg->rt_se); +} + +static int alloc_rt_sched_group (struct task_group *tg) +{ +struct rt_rq *rt_rq; +struct sched_rt_entity *rt_se; +struct rq *rq; +int i; + + tg->rt_rq =kzalloc (sizeof (rt_rq) *NR_CPUS ,GFP_KERNEL ); +if (!tg->rt_rq) + goto err; + tg->rt_se =kzalloc (sizeof (rt_se) *NR_CPUS ,GFP_KERNEL ); +if (!tg->rt_se) + goto err; + + tg->rt_runtime =0 ; + +for_each_possible_cpu (i) { + rq =cpu_rq (i); + rt_rq =kmalloc_node (sizeof (struct rt_rq ),GFP_KERNEL |__GFP_ZERO,cpu_to_node (i));if (!rt_rq)
@@ -7663,24 +7682,55 @@ struct task_group *sched_create_group(vo goto err; init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); -#endif }lock_task_group_list ();for_each_possible_cpu (i) { rq =cpu_rq (i); -#ifdef CONFIG_FAIR_GROUP_SCHED - cfs_rq = tg->cfs_rq[i]; -list_add_rcu (&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); -#endif -#ifdef CONFIG_RT_GROUP_SCHED rt_rq = tg->rt_rq[i];list_add_rcu (&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); -#endif }list_add_rcu (&tg->list, &task_groups);unlock_task_group_list (); +return 1 ; + + err: +return 0 ; +} +#else +static inline void free_rt_sched_group (struct task_group *tg) +{ +} + +static inline int alloc_rt_sched_group (struct task_group *tg) +{ +return 1 ; +} +#endif + +static void free_sched_group (struct task_group *tg) +{ +free_fair_sched_group (tg); +free_rt_sched_group (tg); +kfree (tg); +} + +/* allocate runqueue etc for a new task group */ +struct task_group *sched_create_group (void ) +{ +struct task_group *tg; + + tg =kzalloc (sizeof (*tg),GFP_KERNEL ); +if (!tg) +return ERR_PTR (-ENOMEM ); + +if (!alloc_fair_sched_group (tg)) + goto err; + +if (!alloc_rt_sched_group (tg)) + goto err; +return tg; err:
--
PATCH 6/8 sched: rt-group: refure unrunnable tasks
In reply to a message not in the archiveRefuse to accept or create RT tasks in groups that can't run them.
--- kernel/sched.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
kernel/sched.c +15 -0
@@ -4588,6 +4588,15 @@ recheck:return -EPERM ; } +#ifdef CONFIG_RT_GROUP_SCHED +/* +* Do not allow realtime tasks into groups that have no runtime +* assigned. +*/ +if (rt_policy (policy) &&task_group (p)->rt_runtime ==0 ) +return -EPERM ; +#endif + retval =security_task_setscheduler (p, policy, param);if (retval)return retval;
@@ -8005,9 +8014,15 @@ static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp, struct task_struct *tsk) { +#ifdef CONFIG_RT_GROUP_SCHED + /* Don't accept realtime tasks when there is no way for them to run */ + if (rt_task(tsk) && cgroup_tg(cgrp)->rt_runtime == 0) + return -EINVAL; +#else /* We don't support RT-tasks being in separate groups */ if (tsk->sched_class != &fair_sched_class)return -EINVAL ; +#endif return 0 ; }
--
PATCH 7/8 sched: rt-group: synchonised bandwidth period
In reply to a message not in the archiveVarious SMP balancing algorithms require that the bandwidth period run in sync.
Possible improvements are moving the rt_bandwidth thing into root_domain and keeping a span per rt_bandwidth which marks throttled cpus.
--- include/linux/sched.h | 7 + kernel/sched.c | 263 +++++++++++++++++++++++++++++++++++++---------- kernel/sched_rt.c | 104 ++++++++++++------ kernel/sysctl.c | 4 kernel/time/tick-sched.c | 5 5 files changed, 293 insertions(+), 90 deletions(-)
include/linux/sched.h +7 -0
@@ -1519,6 +1519,10 @@ int sched_nr_latency_handler(struct ctl_extern unsigned int sysctl_sched_rt_period;extern int sysctl_sched_rt_runtime; +int sched_rt_handler (struct ctl_table *table,int write, +struct file *filp,void __user *buffer,size_t *lenp, +loff_t *ppos); +extern unsigned int sysctl_sched_compat_yield;#ifdef CONFIG_RT_MUTEXES
@@ -2000,6 +2004,9 @@ extern unsigned long sched_group_shares(extern int sched_group_set_rt_runtime (struct task_group *tg,long rt_runtime_us);extern long sched_group_rt_runtime (struct task_group *tg); +extern int sched_group_set_rt_period (struct task_group *tg, +long rt_period_us); +extern long sched_group_rt_period (struct task_group *tg);#endif
kernel/sched.c +212 -51
418 lines not on this page · raw message
kernel/sched_rt.c +72 -32
147 lines not on this page · raw message
kernel/sysctl.c +2 -2
@@ -336,7 +336,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_sched_rt_period, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = &proc_dointvec, + .proc_handler = &sched_rt_handler, }, { .ctl_name = CTL_UNNUMBERED,
@@ -344,7 +344,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_sched_rt_runtime, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = &proc_dointvec, + .proc_handler = &sched_rt_handler, }, { .ctl_name = CTL_UNNUMBERED,
kernel/time/tick-sched.c +0 -5
@@ -191,7 +191,6 @@ u64 get_cpu_idle_time_us(int cpu, u64 *lvoid tick_nohz_stop_sched_tick (void ) {unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; -unsigned long rt_jiffies;struct tick_sched *ts;ktime_t last_update, expires, now;struct clock_event_device *dev =__get_cpu_var (tick_cpu_device).evtdev;
@@ -244,10 +243,6 @@ void tick_nohz_stop_sched_tick(void) next_jiffies =get_next_timer_interrupt (last_jiffies); delta_jiffies = next_jiffies - last_jiffies; - rt_jiffies =rt_needs_cpu (cpu); -if (rt_jiffies && rt_jiffies < delta_jiffies) - delta_jiffies = rt_jiffies; -if (rcu_needs_cpu (cpu)) delta_jiffies =1 ;/*
--
PATCH 8/8 sched: rt-group: smp balancing
In reply to a message not in the archiveCurrently the rt group scheduling does a per cpu runtime limit, however the rt load balancer makes no guarantees about an equal spread of real- time tasks, just that at any one time, the highest priority tasks run.
Solve this by making the runtime limit a global property by borrowing excessive runtime from the other cpus once the local limit runs out.
--- kernel/sched.c | 38 ++++++++++++++++++++++- kernel/sched_rt.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 5 deletions(-)
kernel/sched.c +36 -2
@@ -169,6 +169,7 @@ static inline ktime_t ns_to_ktime(u64 nsstruct rt_bandwidth {ktime_t rt_period;u64 rt_runtime; +spinlock_t rt_runtime_lock;struct hrtimer rt_period_timer; };
@@ -203,6 +204,8 @@ void init_rt_bandwidth(struct rt_bandwid rt_b->rt_period =ns_to_ktime (period); rt_b->rt_runtime = runtime; +spin_lock_init (&rt_b->rt_runtime_lock); +hrtimer_init (&rt_b->rt_period_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); rt_b->rt_period_timer.function = sched_rt_period_timer;
@@ -471,6 +474,8 @@ struct rt_rq {#endif int rt_throttled;u64 rt_time; +u64 rt_runtime; +spinlock_t rt_runtime_lock;#ifdef CONFIG_RT_GROUP_SCHED unsigned long rt_nr_boosted;
@@ -7216,6 +7221,8 @@ static void init_rt_rq(struct rt_rq *rt_ rt_rq->rt_time =0 ; rt_rq->rt_throttled =0 ; + rt_rq->rt_runtime =0 ; +spin_lock_init (&rt_rq->rt_runtime_lock);#ifdef CONFIG_RT_GROUP_SCHED rt_rq->rt_nr_boosted =0 ;
@@ -7252,6 +7259,7 @@ static void init_tg_rt_entry(struct rq *init_rt_rq (rt_rq, rq); rt_rq->tg = tg; rt_rq->rt_se = rt_se; + rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;if (add)list_add (&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
@@ -7307,6 +7315,8 @@ void __init sched_init(void)init_tg_rt_entry (rq, &init_task_group, &per_cpu (init_rt_rq, i), &per_cpu (init_sched_rt_entity, i), i,1 ); +#else + rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;#endif for (j =0 ; j <CPU_LOAD_IDX_MAX ; j++)
@@ -8007,15 +8017,26 @@ static int __rt_schedulable(struct task_static int tg_set_bandwidth (struct task_group *tg,u64 rt_period,u64 rt_runtime) { -int err =0 ; +int i, err =0 ;mutex_lock (&rt_constraints_mutex);if (!__rt_schedulable (tg, rt_period, rt_runtime)) { err = -EINVAL ; goto unlock; } + +spin_lock_irq (&tg->rt_bandwidth.rt_runtime_lock); tg->rt_bandwidth.rt_period =ns_to_ktime (rt_period); tg->rt_bandwidth.rt_runtime = rt_runtime; + +for_each_possible_cpu (i) { +struct rt_rq *rt_rq = tg->rt_rq[i]; + +spin_lock (&rt_rq->rt_runtime_lock); + rt_rq->rt_runtime = rt_runtime; +spin_unlock (&rt_rq->rt_runtime_lock); + } +spin_unlock_irq (&tg->rt_bandwidth.rt_runtime_lock); unlock:mutex_unlock (&rt_constraints_mutex);
@@ -8079,6 +8100,19 @@ static int sched_rt_global_constraints(v#else static int sched_rt_global_constraints (void ) { +unsigned long flags; +int i; + +spin_lock_irqsave (&def_rt_bandwidth.rt_runtime_lock, flags); +for_each_possible_cpu (i) { +struct rt_rq *rt_rq = &cpu_rq (i)->rt; + +spin_lock (&rt_rq->rt_runtime_lock); + rt_rq->rt_runtime =global_rt_runtime (); +spin_unlock (&rt_rq->rt_runtime_lock); + } +spin_unlock_irqrestore (&def_rt_bandwidth.rt_runtime_lock, flags); +return 0 ; }#endif
@@ -8195,7 +8229,7 @@ static u64 cpu_shares_read_uint(struct c#endif #ifdef CONFIG_RT_GROUP_SCHED -static int cpu_rt_runtime_write (struct cgroup *cgrp,struct cftype *cft, +static ssize_t cpu_rt_runtime_write (struct cgroup *cgrp,struct cftype *cft,struct file *file,const char __user *userbuf,size_t nbytes,loff_t *unused_ppos)
kernel/sched_rt.c +85 -3
144 lines not on this page · raw message
--