Skip to contentSkip to conversation

sched: rt-group: smp balancing

PATCH In review Peter Zijlstra 3 participants 14 messages Feb 4, 2008 – Feb 23, 2008 1 message not in the archive

Subsystems (5)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH 1/8 sched: fix incorrect irq lock usage in normalize_rt_tasks()

lockdep spotted this bogus irq locking. normalize_rt_tasks() can be called from hardirq context through sysrq-n

---
 kernel/sched.c |   10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
kernel/sched.c +4 -4
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -7345,7 +7345,7 @@ void normalize_rt_tasks(void)
 	unsigned long flags;
 	struct rq *rq;
 
-	read_lock_irq(&tasklist_lock);
+	read_lock_irqsave(&tasklist_lock, flags);
 	do_each_thread(g, p) {
 		/*
 		 * Only normalize user tasks:
@@ -7371,16 +7371,16 @@ void normalize_rt_tasks(void)
 			continue;
 		}
 
-		spin_lock_irqsave(&p->pi_lock, flags);
+		spin_lock(&p->pi_lock);
 		rq = __task_rq_lock(p);
 
 		normalize_task(rq, p);
 
 		__task_rq_unlock(rq);
-		spin_unlock_irqrestore(&p->pi_lock, flags);
+		spin_unlock(&p->pi_lock);
 	} while_each_thread(g, p);
 
-	read_unlock_irq(&tasklist_lock);
+	read_unlock_irqrestore(&tasklist_lock, flags);
 }
 
 #endif /* CONFIG_MAGIC_SYSRQ */

--

#2 lore raw

PATCH 2/8 sched: rt-group: deal with PI

In reply to a message not in the archive

Steven mentioned the fun case where a lock holding task will be throttled.

Simple fix: allow groups that have boosted tasks to run anyway.

If a runnable task in a throttled group gets boosted the dequeue/enqueue done by rt_mutex_setprio() is enough to unthrottle the group.

This is ofcourse not quite correct. Two possible ways forward are:
- second prio array for boosted tasks
- boost to a prio ceiling (this would also work for deadline scheduling)

---
 kernel/sched.c    |    3 +++
 kernel/sched_rt.c |   43 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 41 insertions(+), 5 deletions(-)
kernel/sched.c +3 -0
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -374,6 +374,8 @@ struct rt_rq {
 	u64 rt_time;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
+	unsigned long rt_nr_boosted;
+
 	struct rq *rq;
 	struct list_head leaf_rt_rq_list;
 	struct task_group *tg;
@@ -7116,6 +7118,7 @@ static void init_rt_rq(struct rt_rq *rt_
 	rt_rq->rt_throttled = 0;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
+	rt_rq->rt_nr_boosted = 0;
 	rt_rq->rq = rq;
 #endif
 }
kernel/sched_rt.c +38 -5

91 lines not on this page · raw message

--

#3 lore raw

PATCH 3/8 sched: rt-group: interface

In reply to a message not in the archive

Change the rt_ratio interface to rt_runtime_us, to match rt_period_us. This avoids picking a granularity for the ratio.

Extend the /sys/kernel/uids/<uid>/ interface to allow setting the group's rt_runtime.

---
 Documentation/ABI/testing/sysfs-kernel-uids |    6 +
 Documentation/sched-rt-group.txt            |   59 +++++++++++
 include/linux/sched.h                       |    7 -
 kernel/sched.c                              |  145 +++++++++++++++++++++-------
 kernel/sched_rt.c                           |   53 ++++------
 kernel/sysctl.c                             |   32 +++---
 kernel/user.c                               |   28 +++++
 7 files changed, 250 insertions(+), 80 deletions(-)
include/linux/sched.h +5 -2
Index: linux-2.6/include/linux/sched.h
===================================================================
@@ -1507,8 +1507,6 @@ extern unsigned int sysctl_sched_child_r
 extern unsigned int sysctl_sched_features;
 extern unsigned int sysctl_sched_migration_cost;
 extern unsigned int sysctl_sched_nr_migrate;
-extern unsigned int sysctl_sched_rt_period;
-extern unsigned int sysctl_sched_rt_ratio;
 #if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP)
 extern unsigned int sysctl_sched_min_bal_int_shares;
 extern unsigned int sysctl_sched_max_bal_int_shares;
@@ -1518,6 +1516,8 @@ int sched_nr_latency_handler(struct ctl_
 		struct file *file, void __user *buffer, size_t *length,
 		loff_t *ppos);
 #endif
+extern unsigned int sysctl_sched_rt_period;
+extern int sysctl_sched_rt_runtime;
 
 extern unsigned int sysctl_sched_compat_yield;
 
@@ -1997,6 +1997,9 @@ extern void sched_destroy_group(struct t
 extern void sched_move_task(struct task_struct *tsk);
 extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
 extern unsigned long sched_group_shares(struct task_group *tg);
+extern int sched_group_set_rt_runtime(struct task_group *tg,
+				      long rt_runtime_us);
+extern long sched_group_rt_runtime(struct task_group *tg);
 
 #endif
 
kernel/sched.c +113 -32

209 lines not on this page · raw message

kernel/sched_rt.c +23 -30

126 lines not on this page · raw message

kernel/sysctl.c +16 -16
Index: linux-2.6/kernel/sysctl.c
===================================================================
@@ -311,22 +311,6 @@ static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= &proc_dointvec,
 	},
-	{
-		.ctl_name	= CTL_UNNUMBERED,
-		.procname	= "sched_rt_period_ms",
-		.data		= &sysctl_sched_rt_period,
-		.maxlen		= sizeof(unsigned int),
-		.mode		= 0644,
-		.proc_handler	= &proc_dointvec,
-	},
-	{
-		.ctl_name	= CTL_UNNUMBERED,
-		.procname	= "sched_rt_ratio",
-		.data		= &sysctl_sched_rt_ratio,
-		.maxlen		= sizeof(unsigned int),
-		.mode		= 0644,
-		.proc_handler	= &proc_dointvec,
-	},
 #if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP)
 	{
 		.ctl_name       = CTL_UNNUMBERED,
@@ -348,6 +332,22 @@ static struct ctl_table kern_table[] = {
 #endif
 	{
 		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "sched_rt_period_us",
+		.data		= &sysctl_sched_rt_period,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "sched_rt_runtime_us",
+		.data		= &sysctl_sched_rt_runtime,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+	},
+	{
+		.ctl_name	= CTL_UNNUMBERED,
 		.procname	= "sched_compat_yield",
 		.data		= &sysctl_sched_compat_yield,
 		.maxlen		= sizeof(unsigned int),
Documentation/ABI/testing/sysfs-kernel-uids +6 -0
Index: linux-2.6/Documentation/ABI/testing/sysfs-kernel-uids
===================================================================
@@ -12,3 +12,9 @@ Description:
 		B has shares = 2048, User B will get twice the CPU
 		bandwidth user A will. For more details refer
 		Documentation/sched-design-CFS.txt
+
+What:		/sys/kernel/uids/<uid>/cpu_rt_runtime_us
+Date:		January 2008
+Contact:	Peter Zijlstra <[email protected]>
+Description:	See Documentation/sched-rt-group.txt
+
Documentation/sched-rt-group.txt new file +59 -0

59 lines not on this page · raw message

kernel/user.c +28 -0
Index: linux-2.6/kernel/user.c
===================================================================
@@ -156,9 +156,37 @@ static ssize_t cpu_shares_store(struct k
 static struct kobj_attribute cpu_share_attr =
 	__ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store);
 
+static ssize_t cpu_rt_runtime_show(struct kobject *kobj,
+				   struct kobj_attribute *attr,
+				   char *buf)
+{
+	struct user_struct *up = container_of(kobj, struct user_struct, kobj);
+
+	return sprintf(buf, "%lu\n", sched_group_rt_runtime(up->tg));
+}
+
+static ssize_t cpu_rt_runtime_store(struct kobject *kobj,
+				    struct kobj_attribute *attr,
+				    const char *buf, size_t size)
+{
+	struct user_struct *up = container_of(kobj, struct user_struct, kobj);
+	unsigned long rt_runtime;
+	int rc;
+
+	sscanf(buf, "%lu", &rt_runtime);
+
+	rc = sched_group_set_rt_runtime(up->tg, rt_runtime);
+
+	return (rc ? rc : size);
+}
+
+static struct kobj_attribute cpu_rt_runtime_attr =
+	__ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store);
+
 /* default attributes per uid directory */
 static struct attribute *uids_attributes[] = {
 	&cpu_share_attr.attr,
+	&cpu_rt_runtime_attr.attr,
 	NULL
 };
 

--

#4 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

Quoted 28 lines · On Mon, 04 Feb 2008 22:03:01 +0100 Peter Zijlstra wrote:

Change the rt_ratio interface to rt_runtime_us, to match rt_period_us. This avoids picking a granularity for the ratio.

Extend the /sys/kernel/uids/<uid>/ interface to allow setting the group's rt_runtime.

---
 Documentation/ABI/testing/sysfs-kernel-uids |    6 +
 Documentation/sched-rt-group.txt            |   59 +++++++++++
 include/linux/sched.h                       |    7 -
 kernel/sched.c                              |  145 +++++++++++++++++++++-------
 kernel/sched_rt.c                           |   53 ++++------
 kernel/sysctl.c                             |   32 +++---
 kernel/user.c                               |   28 +++++
 7 files changed, 250 insertions(+), 80 deletions(-)

Index: linux-2.6/kernel/sched.c
===================================================================

kernel/sched.c +1 -1
@@ -7780,30 +7783,76 @@ unsigned long sched_group_shares(struct 
 }
 
 /*
- * Ensure the total rt_ratio <= sysctl_sched_rt_ratio
+ * Ensure that the real time constraints are schedulable.
  */

kernel/sched.c unsigned long sched_group_shares(struct · line 7796

Quoted 10 lines
kernel/sched.c +9 -1
-int sched_group_set_rt_ratio(struct task_group *tg, unsigned long rt_ratio)
+static DEFINE_MUTEX(rt_constraints_mutex);
+
+static unsigned long to_ratio(u64 period, u64 runtime)
+{
+	if (runtime == RUNTIME_INF)
+		return 1ULL << 16;
+
+	runtime *= (1ULL << 16);
+	do_div(runtime, period);

Isn't do_div() defined as taking (uint64_t, uint32_t) ?

Quoted 9 lines
kernel/sched.c +3 -0
+	return runtime;
+}
+
 

Index: linux-2.6/Documentation/sched-rt-group.txt
===================================================================

Documentation/sched-rt-group.txt line 8

Quoted 8 lines
Documentation/sched-rt-group.txt new file +8 -0
@@ -0,0 +1,59 @@
+
+
+Real-Time group scheduling.
+
+The problem space:
+
+In order to schedule multiple groups of realtime tasks each group must
+be assigned a fixed portion of the cpu time available. Without a minimum

Use "cpu" or "CPU" consistently, please. (I prefer CPU, but ....)

Documentation/sched-rt-group.txt line 13

Quoted 5 lines
Documentation/sched-rt-group.txt new file +5 -0
+guarantee a realtime group can obviously fall short. A fuzzy upper limit
+is of no use since it cannot be relied upon. Which leaves us with just
+the single fixed portion.
+
+CPU time is divided by means of specifying how much time can be spend

s/spend/spent/

Documentation/sched-rt-group.txt line 15

Quoted 2 lines
Documentation/sched-rt-group.txt new file +2 -0
+running in a given period. Say a frame fixed realtime renderer must
+deliver a 25 frames a second, which yields a period of 0.04s. Now say

drop "a"^

Documentation/sched-rt-group.txt line 23

Quoted 8 lines
Documentation/sched-rt-group.txt new file +8 -0
+it will also have to play some music and respond to input, leaving it
+with around 80% for the graphics. We can then give this group a runtime
+of 0.8 * 0.04s = 0.032s.
+
+This way the graphics group will have a 0.04s period with a 0.032s runtime
+limit.
+
+Now if the audio thread needs to refill the dma buffer every 0.005s, but

DMA preferably.

Documentation/sched-rt-group.txt line 24

Quoted 1 line
Documentation/sched-rt-group.txt new file +1 -0
+needs only about 3% cpu time to do so, it will can do with a 0.03 * 0.005s

s/will can do/can do/

Quoted 11 lines
Documentation/sched-rt-group.txt new file +11 -0
+= 0.00015s.
+
+
+The Interface:
+
+system wide:
+
+/proc/sys/kernel/sched_rt_period_ms
+/proc/sys/kernel/sched_rt_runtime_us
+
+CONFIG_FAIR_USER_SCHED

Documentation/sched-rt-group.txt line 45

Quoted 10 lines
Documentation/sched-rt-group.txt new file +10 -0
+
+/sys/kernel/uids/<uid>/cpu_rt_runtime_us
+
+or
+
+CONFIG_FAIR_CGROUP_SCHED
+
+/cgroup/<cgroup>/cpu.rt_runtime_us
+
+[ time is specified in us because the interface is s32, this gives an

s/,/;/

Documentation/sched-rt-group.txt line 55

Quoted 10 lines
Documentation/sched-rt-group.txt new file +10 -0
+  operating range of ~35m to 1us ]
+
+The period takes values in [ 1, INT_MAX ], runtime in [ -1, INT_MAX - 1 ].
+
+A runtime of -1 specifies runtime == period, ie. no limit.
+
+New groups get the period from /proc/sys/kernel/sched_rt_period_us and
+a runtime of 0.
+
+Settings are constrainted to:

constrained

Documentation/sched-rt-group.txt line 59

Quoted 4 lines
Documentation/sched-rt-group.txt new file +4 -0
+
+   \Sum_{i} runtime_{i} / global_period <= global_runtime / global_period
+
+in order to keep the configuration schedulable.

---
~Randy

#5 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

In reply to Peter Zijlstra
Quoted 45 lines · Peter Zijlstra

+static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,

 +                               struct file *file,
 +                               const char __user *userbuf,
 +                               size_t nbytes, loff_t *unused_ppos)
 +{
 +       char buffer[64];
 +       int retval = 0;
 +       s64 val;
 +       char *end;
 +
 +       if (!nbytes)
 +               return -EINVAL;
 +       if (nbytes >= sizeof(buffer))
 +               return -E2BIG;
 +       if (copy_from_user(buffer, userbuf, nbytes))
 +               return -EFAULT;
 +
 +       buffer[nbytes] = 0;     /* nul-terminate */
 +
 +       /* strip newline if necessary */
 +       if (nbytes && (buffer[nbytes-1] == '\n'))
 +               buffer[nbytes-1] = 0;
 +       val = simple_strtoll(buffer, &end, 0);
 +       if (*end)
 +               return -EINVAL;
 +
 +       /* Pass to subsystem */
 +       retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
 +       if (!retval)

+ retval = nbytes;
+ return retval;
}

-static u64 cpu_rt_ratio_read_uint(struct cgroup *cgrp, struct cftype *cft) -{
- struct task_group *tg = cgroup_tg(cgrp);
+static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,

 +                                  struct file *file,
 +                                  char __user *buf, size_t nbytes,
 +                                  loff_t *ppos)
 +{
 +       char tmp[64];
 +       long val = sched_group_rt_runtime(cgroup_tg(cgrp));
 +       int len = sprintf(tmp, "%ld\n", val);
Quoted 3 lines

- return (u64) tg->rt_ratio;
+ return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
}

What's the reason that you can't use the cgroup read_uint/write_uint methods for this? Is it just because you have -1 as your "unlimited" value.

If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.

Paul

#6 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

Quoted 46 lines · Paul Menage
Quoted 45 lines · Peter Zijlstra

+static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,

 +                               struct file *file,
 +                               const char __user *userbuf,
 +                               size_t nbytes, loff_t *unused_ppos)
 +{
 +       char buffer[64];
 +       int retval = 0;
 +       s64 val;
 +       char *end;
 +
 +       if (!nbytes)
 +               return -EINVAL;
 +       if (nbytes >= sizeof(buffer))
 +               return -E2BIG;
 +       if (copy_from_user(buffer, userbuf, nbytes))
 +               return -EFAULT;
 +
 +       buffer[nbytes] = 0;     /* nul-terminate */
 +
 +       /* strip newline if necessary */
 +       if (nbytes && (buffer[nbytes-1] == '\n'))
 +               buffer[nbytes-1] = 0;
 +       val = simple_strtoll(buffer, &end, 0);
 +       if (*end)
 +               return -EINVAL;
 +
 +       /* Pass to subsystem */
 +       retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
 +       if (!retval)

+ retval = nbytes;
+ return retval;
}

-static u64 cpu_rt_ratio_read_uint(struct cgroup *cgrp, struct cftype *cft) -{
- struct task_group *tg = cgroup_tg(cgrp);
+static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,

 +                                  struct file *file,
 +                                  char __user *buf, size_t nbytes,
 +                                  loff_t *ppos)
 +{
 +       char tmp[64];
 +       long val = sched_group_rt_runtime(cgroup_tg(cgrp));
 +       int len = sprintf(tmp, "%ld\n", val);
Quoted 7 lines
Quoted 3 lines

- return (u64) tg->rt_ratio;
+ return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
}

What's the reason that you can't use the cgroup read_uint/write_uint methods for this? Is it just because you have -1 as your "unlimited" value.

Yes.

Quoted 3 lines

If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.

0 means no time, quite useful and clearly distinct from inf. time.

#7 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

Quoted 6 lines · Peter Zijlstra
Quoted 3 lines

If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.

0 means no time, quite useful and clearly distinct from inf. time.

So a real-time task in a cgroup with a 0 rt_runtime can be in the R state but never actually get to run? OK, if people need to be able to do that then fair enough.

In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.

Paul

#8 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

Quoted 11 lines · Paul Menage
Quoted 6 lines · Peter Zijlstra
Quoted 3 lines

If so, could we avoid that problem by using 0 rather than -1 as the "unlimited" value? It looks from what I've read in the Documentation changes as though 0 isn't really a meaningful value.

0 means no time, quite useful and clearly distinct from inf. time.

So a real-time task in a cgroup with a 0 rt_runtime can be in the R state but never actually get to run? OK, if people need to be able to do that then fair enough.

Yeah, its an awkward situation, and we refuse new rt tasks in such groups. But the 0 value is needed so you can have groups that don't participate in the realtime scheduling because we enforce a schedulalbility constraint over the groups.

Each group has a runtime ratio, namely: rt_runtime / rt_period. The sum of this ratio over all groups must be smaller or equal to the global ratio which must be smaller or equal to 1.

Quoted 2 lines

In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.

Yes, I looked at that, I found the interface somewhat unfortunate, it would mean growing the struct with two more function pointers. Perhaps a read and write function with abstract data would be better suited. That would allow for this and more. Sadly it looses type information.

#9 lore raw

Re: [PATCH 3/8] sched: rt-group: interface

Quoted 6 lines · Peter Zijlstra
Quoted 3 lines

In that case I guess I'll have to add signed versions of the read_uint/write_uint methods.

Yes, I looked at that, I found the interface somewhat unfortunate, it would mean growing the struct with two more function pointers.

Is that really a big deal? We're talking about a structure that has a small number (<10 in the current tree) of instances per cgroup subsystem.

Quoted 3 lines

Perhaps a
read and write function with abstract data would be better suited. That would allow for this and more. Sadly it looses type information.

If the size of the struct cftype really became a problem, I think the cleanest way to fix it would be to have a union of the potential function pointers, and add a field to specify which one is in use.

Paul

#10 lore raw

PATCH 4/8 sched: rt-group: make rt groups scheduling configurable

In reply to a message not in the archive

Make the rt group scheduler compile time configurable. Enable it by default for cgroup scheduling.

---
 include/linux/cgroup_subsys.h |    2 
 init/Kconfig                  |   23 +++++--
 kernel/sched.c                |  130 +++++++++++++++++++++++++++++++-----------
 kernel/sched_rt.c             |   12 +--
 4 files changed, 120 insertions(+), 47 deletions(-)
include/linux/cgroup_subsys.h +1 -1
Index: linux-2.6/include/linux/cgroup_subsys.h
===================================================================
@@ -25,7 +25,7 @@ SUBSYS(ns)
 
 /* */
 
-#ifdef CONFIG_FAIR_CGROUP_SCHED
+#ifdef CONFIG_CGROUP_SCHED
 SUBSYS(cpu_cgroup)
 #endif
 
init/Kconfig +17 -6
Index: linux-2.6/init/Kconfig
===================================================================
@@ -332,25 +332,36 @@ config CPUSETS
 
 	  Say N if unsure.
 
-config FAIR_GROUP_SCHED
-	bool "Fair group CPU scheduler"
+config GROUP_SCHED
+	bool "Group CPU scheduler"
 	default y
 	help
 	  This feature lets CPU scheduler recognize task groups and control CPU
 	  bandwidth allocation to such task groups.
 
+config FAIR_GROUP_SCHED
+	bool "Group scheduling for SCHED_OTHER"
+	depends on GROUP_SCHED
+	default y
+
+config RT_GROUP_SCHED
+	bool "Group scheduling for SCHED_RR/FIFO"
+	depends on EXPERIMENTAL
+	depends on GROUP_SCHED
+	default n
+
 choice
-	depends on FAIR_GROUP_SCHED
+	depends on GROUP_SCHED
 	prompt "Basis for grouping tasks"
-	default FAIR_USER_SCHED
+	default USER_SCHED
 
-config FAIR_USER_SCHED
+config USER_SCHED
 	bool "user id"
 	help
 	  This option will choose userid as the basis for grouping
 	  tasks, thus providing equal CPU bandwidth to each user.
 
-config FAIR_CGROUP_SCHED
+config CGROUP_SCHED
 	bool "Control groups"
  	depends on CGROUPS
  	help
kernel/sched.c +96 -34

420 lines not on this page · raw message

kernel/sched_rt.c +6 -6
Index: linux-2.6/kernel/sched_rt.c
===================================================================
@@ -55,7 +55,7 @@ static inline int on_rt_rq(struct sched_
 	return !list_empty(&rt_se->run_list);
 }
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
+#ifdef CONFIG_RT_GROUP_SCHED
 
 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
 {
@@ -177,7 +177,7 @@ static inline int rt_rq_throttled(struct
 
 static inline int rt_se_prio(struct sched_rt_entity *rt_se)
 {
-#ifdef CONFIG_FAIR_GROUP_SCHED
+#ifdef CONFIG_RT_GROUP_SCHED
 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
 
 	if (rt_rq)
@@ -269,7 +269,7 @@ void inc_rt_tasks(struct sched_rt_entity
 {
 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
 	rt_rq->rt_nr_running++;
-#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED
+#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
 	if (rt_se_prio(rt_se) < rt_rq->highest_prio)
 		rt_rq->highest_prio = rt_se_prio(rt_se);
 #endif
@@ -281,7 +281,7 @@ void inc_rt_tasks(struct sched_rt_entity
 
 	update_rt_migration(rq_of_rt_rq(rt_rq));
 #endif
-#ifdef CONFIG_FAIR_GROUP_SCHED
+#ifdef CONFIG_RT_GROUP_SCHED
 	if (rt_se_boosted(rt_se))
 		rt_rq->rt_nr_boosted++;
 #endif
@@ -293,7 +293,7 @@ void dec_rt_tasks(struct sched_rt_entity
 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
 	WARN_ON(!rt_rq->rt_nr_running);
 	rt_rq->rt_nr_running--;
-#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED
+#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
 	if (rt_rq->rt_nr_running) {
 		struct rt_prio_array *array;
 
@@ -315,7 +315,7 @@ void dec_rt_tasks(struct sched_rt_entity
 
 	update_rt_migration(rq_of_rt_rq(rt_rq));
 #endif /* CONFIG_SMP */
-#ifdef CONFIG_FAIR_GROUP_SCHED
+#ifdef CONFIG_RT_GROUP_SCHED
 	if (rt_se_boosted(rt_se))
 		rt_rq->rt_nr_boosted--;
 

--

#11 lore raw

PATCH 5/8 sched: rt-group: clean up the ifdeffery

In reply to a message not in the archive

Clean up some of the excessive ifdeffery introduces in the last patch.

---
 kernel/sched.c |  150 ++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 100 insertions(+), 50 deletions(-)
kernel/sched.c +100 -50
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -7563,56 +7563,29 @@ static int load_balance_monitor(void *un
 }
 #endif	/* CONFIG_SMP */
 
-static void free_sched_group(struct task_group *tg)
+#ifdef CONFIG_FAIR_GROUP_SCHED
+static void free_fair_sched_group(struct task_group *tg)
 {
 	int i;
 
 	for_each_possible_cpu(i) {
-#ifdef CONFIG_FAIR_GROUP_SCHED
 		if (tg->cfs_rq)
 			kfree(tg->cfs_rq[i]);
 		if (tg->se)
 			kfree(tg->se[i]);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
-		if (tg->rt_rq)
-			kfree(tg->rt_rq[i]);
-		if (tg->rt_se)
-			kfree(tg->rt_se[i]);
-#endif
 	}
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
 	kfree(tg->cfs_rq);
 	kfree(tg->se);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
-	kfree(tg->rt_rq);
-	kfree(tg->rt_se);
-#endif
-	kfree(tg);
 }
 
-/* allocate runqueue etc for a new task group */
-struct task_group *sched_create_group(void)
+static int alloc_fair_sched_group(struct task_group *tg)
 {
-	struct task_group *tg;
-#ifdef CONFIG_FAIR_GROUP_SCHED
 	struct cfs_rq *cfs_rq;
 	struct sched_entity *se;
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
-	struct rt_rq *rt_rq;
-	struct sched_rt_entity *rt_se;
-#endif
 	struct rq *rq;
 	int i;
 
-	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
-	if (!tg)
-		return ERR_PTR(-ENOMEM);
-
-#ifdef CONFIG_FAIR_GROUP_SCHED
 	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL);
 	if (!tg->cfs_rq)
 		goto err;
@@ -7621,23 +7594,10 @@ struct task_group *sched_create_group(vo
 		goto err;
 
 	tg->shares = NICE_0_LOAD;
-#endif
-
-#ifdef CONFIG_RT_GROUP_SCHED
-	tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL);
-	if (!tg->rt_rq)
-		goto err;
-	tg->rt_se = kzalloc(sizeof(rt_se) * NR_CPUS, GFP_KERNEL);
-	if (!tg->rt_se)
-		goto err;
-
-	tg->rt_runtime = 0;
-#endif
 
 	for_each_possible_cpu(i) {
 		rq = cpu_rq(i);
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
 		cfs_rq = kmalloc_node(sizeof(struct cfs_rq),
 				GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
 		if (!cfs_rq)
@@ -7649,9 +7609,68 @@ struct task_group *sched_create_group(vo
 			goto err;
 
 		init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
+	}
+
+	lock_task_group_list();
+	for_each_possible_cpu(i) {
+		rq = cpu_rq(i);
+		cfs_rq = tg->cfs_rq[i];
+		list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
+	}
+	list_add_rcu(&tg->list, &task_groups);
+	unlock_task_group_list();
+
+	return 1;
+
+ err:
+	return 0;
+}
+#else
+static inline void free_fair_sched_group(struct task_group *tg)
+{
+}
+
+static inline int alloc_fair_sched_group(struct task_group *tg)
+{
+	return 1;
+}
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
+static void free_rt_sched_group(struct task_group *tg)
+{
+	int i;
+
+	for_each_possible_cpu(i) {
+		if (tg->rt_rq)
+			kfree(tg->rt_rq[i]);
+		if (tg->rt_se)
+			kfree(tg->rt_se[i]);
+	}
+
+	kfree(tg->rt_rq);
+	kfree(tg->rt_se);
+}
+
+static int alloc_rt_sched_group(struct task_group *tg)
+{
+	struct rt_rq *rt_rq;
+	struct sched_rt_entity *rt_se;
+	struct rq *rq;
+	int i;
+
+	tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL);
+	if (!tg->rt_rq)
+		goto err;
+	tg->rt_se = kzalloc(sizeof(rt_se) * NR_CPUS, GFP_KERNEL);
+	if (!tg->rt_se)
+		goto err;
+
+	tg->rt_runtime = 0;
+
+	for_each_possible_cpu(i) {
+		rq = cpu_rq(i);
+
 		rt_rq = kmalloc_node(sizeof(struct rt_rq),
 				GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
 		if (!rt_rq)
@@ -7663,24 +7682,55 @@ struct task_group *sched_create_group(vo
 			goto err;
 
 		init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0);
-#endif
 	}
 
 	lock_task_group_list();
 	for_each_possible_cpu(i) {
 		rq = cpu_rq(i);
-#ifdef CONFIG_FAIR_GROUP_SCHED
-		cfs_rq = tg->cfs_rq[i];
-		list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
 		rt_rq = tg->rt_rq[i];
 		list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
-#endif
 	}
 	list_add_rcu(&tg->list, &task_groups);
 	unlock_task_group_list();
 
+	return 1;
+
+ err:
+	return 0;
+}
+#else
+static inline void free_rt_sched_group(struct task_group *tg)
+{
+}
+
+static inline int alloc_rt_sched_group(struct task_group *tg)
+{
+	return 1;
+}
+#endif
+
+static void free_sched_group(struct task_group *tg)
+{
+	free_fair_sched_group(tg);
+	free_rt_sched_group(tg);
+	kfree(tg);
+}
+
+/* allocate runqueue etc for a new task group */
+struct task_group *sched_create_group(void)
+{
+	struct task_group *tg;
+
+	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
+	if (!tg)
+		return ERR_PTR(-ENOMEM);
+
+	if (!alloc_fair_sched_group(tg))
+		goto err;
+
+	if (!alloc_rt_sched_group(tg))
+		goto err;
+
 	return tg;
 
 err:

--

#12 lore raw

PATCH 6/8 sched: rt-group: refure unrunnable tasks

In reply to a message not in the archive

Refuse to accept or create RT tasks in groups that can't run them.

---
 kernel/sched.c |   15 +++++++++++++++
 1 file changed, 15 insertions(+)
kernel/sched.c +15 -0
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -4588,6 +4588,15 @@ recheck:
 			return -EPERM;
 	}
 
+#ifdef CONFIG_RT_GROUP_SCHED
+	/*
+	 * Do not allow realtime tasks into groups that have no runtime
+	 * assigned.
+	 */
+	if (rt_policy(policy) && task_group(p)->rt_runtime == 0)
+		return -EPERM;
+#endif
+
 	retval = security_task_setscheduler(p, policy, param);
 	if (retval)
 		return retval;
@@ -8005,9 +8014,15 @@ static int
 cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
 		      struct task_struct *tsk)
 {
+#ifdef CONFIG_RT_GROUP_SCHED
+	/* Don't accept realtime tasks when there is no way for them to run */
+	if (rt_task(tsk) && cgroup_tg(cgrp)->rt_runtime == 0)
+		return -EINVAL;
+#else
 	/* We don't support RT-tasks being in separate groups */
 	if (tsk->sched_class != &fair_sched_class)
 		return -EINVAL;
+#endif
 
 	return 0;
 }

--

#13 lore raw

PATCH 7/8 sched: rt-group: synchonised bandwidth period

In reply to a message not in the archive

Various SMP balancing algorithms require that the bandwidth period run in sync.

Possible improvements are moving the rt_bandwidth thing into root_domain and keeping a span per rt_bandwidth which marks throttled cpus.

---
 include/linux/sched.h    |    7 +
 kernel/sched.c           |  263 +++++++++++++++++++++++++++++++++++++----------
 kernel/sched_rt.c        |  104 ++++++++++++------
 kernel/sysctl.c          |    4 
 kernel/time/tick-sched.c |    5 
 5 files changed, 293 insertions(+), 90 deletions(-)
include/linux/sched.h +7 -0
Index: linux-2.6/include/linux/sched.h
===================================================================
@@ -1519,6 +1519,10 @@ int sched_nr_latency_handler(struct ctl_
 extern unsigned int sysctl_sched_rt_period;
 extern int sysctl_sched_rt_runtime;
 
+int sched_rt_handler(struct ctl_table *table, int write,
+		struct file *filp, void __user *buffer, size_t *lenp,
+		loff_t *ppos);
+
 extern unsigned int sysctl_sched_compat_yield;
 
 #ifdef CONFIG_RT_MUTEXES
@@ -2000,6 +2004,9 @@ extern unsigned long sched_group_shares(
 extern int sched_group_set_rt_runtime(struct task_group *tg,
 				      long rt_runtime_us);
 extern long sched_group_rt_runtime(struct task_group *tg);
+extern int sched_group_set_rt_period(struct task_group *tg,
+				      long rt_period_us);
+extern long sched_group_rt_period(struct task_group *tg);
 
 #endif
 
kernel/sched.c +212 -51

418 lines not on this page · raw message

kernel/sched_rt.c +72 -32

147 lines not on this page · raw message

kernel/sysctl.c +2 -2
Index: linux-2.6/kernel/sysctl.c
===================================================================
@@ -336,7 +336,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &sysctl_sched_rt_period,
 		.maxlen		= sizeof(unsigned int),
 		.mode		= 0644,
-		.proc_handler	= &proc_dointvec,
+		.proc_handler	= &sched_rt_handler,
 	},
 	{
 		.ctl_name	= CTL_UNNUMBERED,
@@ -344,7 +344,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &sysctl_sched_rt_runtime,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
-		.proc_handler	= &proc_dointvec,
+		.proc_handler	= &sched_rt_handler,
 	},
 	{
 		.ctl_name	= CTL_UNNUMBERED,
kernel/time/tick-sched.c +0 -5
Index: linux-2.6/kernel/time/tick-sched.c
===================================================================
@@ -191,7 +191,6 @@ u64 get_cpu_idle_time_us(int cpu, u64 *l
 void tick_nohz_stop_sched_tick(void)
 {
 	unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
-	unsigned long rt_jiffies;
 	struct tick_sched *ts;
 	ktime_t last_update, expires, now;
 	struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
@@ -244,10 +243,6 @@ void tick_nohz_stop_sched_tick(void)
 	next_jiffies = get_next_timer_interrupt(last_jiffies);
 	delta_jiffies = next_jiffies - last_jiffies;
 
-	rt_jiffies = rt_needs_cpu(cpu);
-	if (rt_jiffies && rt_jiffies < delta_jiffies)
-		delta_jiffies = rt_jiffies;
-
 	if (rcu_needs_cpu(cpu))
 		delta_jiffies = 1;
 	/*

--

#14 lore raw

PATCH 8/8 sched: rt-group: smp balancing

In reply to a message not in the archive

Currently the rt group scheduling does a per cpu runtime limit, however the rt load balancer makes no guarantees about an equal spread of real- time tasks, just that at any one time, the highest priority tasks run.

Solve this by making the runtime limit a global property by borrowing excessive runtime from the other cpus once the local limit runs out.

---
 kernel/sched.c    |   38 ++++++++++++++++++++++-
 kernel/sched_rt.c |   88 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 121 insertions(+), 5 deletions(-)
kernel/sched.c +36 -2
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -169,6 +169,7 @@ static inline ktime_t ns_to_ktime(u64 ns
 struct rt_bandwidth {
 	ktime_t rt_period;
 	u64 rt_runtime;
+	spinlock_t rt_runtime_lock;
 	struct hrtimer rt_period_timer;
 };
 
@@ -203,6 +204,8 @@ void init_rt_bandwidth(struct rt_bandwid
 	rt_b->rt_period = ns_to_ktime(period);
 	rt_b->rt_runtime = runtime;
 
+	spin_lock_init(&rt_b->rt_runtime_lock);
+
 	hrtimer_init(&rt_b->rt_period_timer,
 			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	rt_b->rt_period_timer.function = sched_rt_period_timer;
@@ -471,6 +474,8 @@ struct rt_rq {
 #endif
 	int rt_throttled;
 	u64 rt_time;
+	u64 rt_runtime;
+	spinlock_t rt_runtime_lock;
 
 #ifdef CONFIG_RT_GROUP_SCHED
 	unsigned long rt_nr_boosted;
@@ -7216,6 +7221,8 @@ static void init_rt_rq(struct rt_rq *rt_
 
 	rt_rq->rt_time = 0;
 	rt_rq->rt_throttled = 0;
+	rt_rq->rt_runtime = 0;
+	spin_lock_init(&rt_rq->rt_runtime_lock);
 
 #ifdef CONFIG_RT_GROUP_SCHED
 	rt_rq->rt_nr_boosted = 0;
@@ -7252,6 +7259,7 @@ static void init_tg_rt_entry(struct rq *
 	init_rt_rq(rt_rq, rq);
 	rt_rq->tg = tg;
 	rt_rq->rt_se = rt_se;
+	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
 	if (add)
 		list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
 
@@ -7307,6 +7315,8 @@ void __init sched_init(void)
 		init_tg_rt_entry(rq, &init_task_group,
 				&per_cpu(init_rt_rq, i),
 				&per_cpu(init_sched_rt_entity, i), i, 1);
+#else
+		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
 #endif
 
 		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
@@ -8007,15 +8017,26 @@ static int __rt_schedulable(struct task_
 static int tg_set_bandwidth(struct task_group *tg,
 		u64 rt_period, u64 rt_runtime)
 {
-	int err = 0;
+	int i, err = 0;
 
 	mutex_lock(&rt_constraints_mutex);
 	if (!__rt_schedulable(tg, rt_period, rt_runtime)) {
 		err = -EINVAL;
 		goto unlock;
 	}
+
+	spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
 	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
 	tg->rt_bandwidth.rt_runtime = rt_runtime;
+
+	for_each_possible_cpu(i) {
+		struct rt_rq *rt_rq = tg->rt_rq[i];
+
+		spin_lock(&rt_rq->rt_runtime_lock);
+		rt_rq->rt_runtime = rt_runtime;
+		spin_unlock(&rt_rq->rt_runtime_lock);
+	}
+	spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  unlock:
 	mutex_unlock(&rt_constraints_mutex);
 
@@ -8079,6 +8100,19 @@ static int sched_rt_global_constraints(v
 #else
 static int sched_rt_global_constraints(void)
 {
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
+	for_each_possible_cpu(i) {
+		struct rt_rq *rt_rq = &cpu_rq(i)->rt;
+
+		spin_lock(&rt_rq->rt_runtime_lock);
+		rt_rq->rt_runtime = global_rt_runtime();
+		spin_unlock(&rt_rq->rt_runtime_lock);
+	}
+	spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
+
 	return 0;
 }
 #endif
@@ -8195,7 +8229,7 @@ static u64 cpu_shares_read_uint(struct c
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
-static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
+static ssize_t cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
 				struct file *file,
 				const char __user *userbuf,
 				size_t nbytes, loff_t *unused_ppos)
kernel/sched_rt.c +85 -3

144 lines not on this page · raw message

--