sched: rt bandwidth/group fixes
cover sched: rt bandwidth/group fixes
my current queue of fixes
1-4 are pushed into sched-urgent and should be included in .26 the rest are less critical and should be queued for .27
PATCH 1/10 sched: NULL pointer dereference while setting sched_rt_period_us
Hi all,
When CONFIG_RT_GROUP_SCHED and CONFIG_CGROUP_SCHED are enabled, with:
echo 10000 > /proc/sys/kernel/sched_rt_period_us
We get this:
BUG: unable to handle kernel NULL pointer dereference at 0000008c [ 947.682233] IP: [<c0216b72>] __rt_schedulable+0x12/0x160 [ 947.683123] *pde = 00000000=20 [ 947.683782] Oops: 0000 [#1] [ 947.684307] Modules linked in: [ 947.684308] [ 947.684308] Pid: 2359, comm: bash Not tainted (2.6.26-rc6 #8) [ 947.684308] EIP: 0060:[<c0216b72>] EFLAGS: 00000246 CPU: 0 [ 947.684308] EIP is at __rt_schedulable+0x12/0x160 [ 947.684308] EAX: 00000000 EBX: 00000000 ECX: 00000000 EDX: 00000001 [ 947.684308] ESI: c0521db4 EDI: 00000001 EBP: c6cc9f00 ESP: c6cc9ed0 [ 947.684308] DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068 [ 947.684308] Process bash (pid: 2359, ti�cc8000 task�a54f00=20 task.ti�cc8000) [ 947.684308] Stack: c0222790 00000000 080f8c08 c0521db4 c6cc9f00 00000001 00000000 00000000 [ 947.684308] c6cc9f9c 00000000 c0521db4 00000001 c6cc9f28 c0216d40 00000000 00000000 [ 947.684308] c6cc9f9c 000f4240 000e7ef0 ffffffff c0521db4 c79dfb60 c6cc9f58 c02af2cc [ 947.684308] Call Trace: [ 947.684308] [<c0222790>] ? do_proc_dointvec_conv+0x0/0x50 [ 947.684308] [<c0216d40>] ? sched_rt_handler+0x80/0x110 [ 947.684308] [<c02af2cc>] ? proc_sys_call_handler+0x9c/0xb0 [ 947.684308] [<c02af2fa>] ? proc_sys_write+0x1a/0x20 [ 947.684308] [<c0273c36>] ? vfs_write+0x96/0x160 [ 947.684308] [<c02af2e0>] ? proc_sys_write+0x0/0x20 [ 947.684308] [<c027423d>] ? sys_write+0x3d/0x70 [ 947.684308] [<c0202ef5>] ? sysenter_past_esp+0x6a/0x91 [ 947.684308] ======================= [ 947.684308] Code: 24 04 e8 62 b1 0e 00 89 c7 89 f8 8b 5d f4 8b 75 f8 8b 7d fc 89 ec 5d c3 90 55 89 e5 57 56 53 83 ec 24 89 45 ec 89 55 e4 89 4d e8 <8b> b8 8c 00 00 00 85 ff 0f 84 c9 00 00 00 8b 57 24 39 55 e8 8b [ 947.684308] EIP: [<c0216b72>] __rt_schedulable+0x12/0x160 SS:ESP 0068:c6cc9ed0
We think the following patch solves the issue.
Hope this is of some help.
Regards,
Dario Faggioli
--- kernel/sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/sched.c +1 -1
@@ -8348,7 +8348,7 @@ static unsigned long to_ratio(u64 period#ifdef CONFIG_CGROUP_SCHED static int __rt_schedulable (struct task_group *tg,u64 period,u64 runtime) { -struct task_group *tgi, *parent = tg->parent; +struct task_group *tgi, *parent = tg ? tg->parent :NULL ;unsigned long total =0 ;if (!parent) {
PATCH 2/10 sched: rt-group: fix hierarchy
In reply to Peter ZijlstraDon't re-set the entity's runqueue to the wrong rq after we've set it to the right one.
--- kernel/sched.c | 1 - 1 file changed, 1 deletion(-)
kernel/sched.c +0 -1
@@ -7626,7 +7626,6 @@ static void init_tg_rt_entry(struct taskelse rt_se->rt_rq = parent->my_q; - rt_se->rt_rq = &rq->rt; rt_se->my_q = rt_rq; rt_se->parent = parent;INIT_LIST_HEAD (&rt_se->run_list);
PATCH 3/10 sched: rt-group: heirarchy aware throttle
In reply to Peter ZijlstraThe bandwidth throttle code dequeues a group when it runs out of quota, and re-queues it once the period rolls over and the quota gets refreshed.
Sadly it failed to take the hierarchy into consideration. Share more of the enqueue/dequeue code with regular task opterations.
Also, some operations like sched_setscheduler() can dequeue/enqueue tasks that are in throttled runqueues, we should not inadvertly re-enqueue empty runqueues so check for that.
--- kernel/sched_rt.c | 59 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 26 deletions(-)
kernel/sched_rt.c +33 -26
@@ -449,13 +449,19 @@ void dec_rt_tasks(struct sched_rt_entity#endif } -static void enqueue_rt_entity (struct sched_rt_entity *rt_se) +static void __enqueue_rt_entity (struct sched_rt_entity *rt_se) {struct rt_rq *rt_rq =rt_rq_of_se (rt_se);struct rt_prio_array *array = &rt_rq->active;struct rt_rq *group_rq =group_rt_rq (rt_se); -if (group_rq &&rt_rq_throttled (group_rq)) +/* +* Don't enqueue the group if its throttled, or when empty. +* The latter is a consequence of the former when a child group +* get throttled and the current group doesn't have any other +* active members. +*/ +if (group_rq && (rt_rq_throttled (group_rq) || !group_rq->rt_nr_running))return ;list_add_tail (&rt_se->run_list, array->queue +rt_se_prio (rt_se));
@@ -464,7 +470,7 @@ static void enqueue_rt_entity(struct schinc_rt_tasks (rt_se, rt_rq); } -static void dequeue_rt_entity (struct sched_rt_entity *rt_se) +static void __dequeue_rt_entity (struct sched_rt_entity *rt_se) {struct rt_rq *rt_rq =rt_rq_of_se (rt_se);struct rt_prio_array *array = &rt_rq->active;
@@ -480,11 +486,10 @@ static void dequeue_rt_entity(struct sch* Because the prio of an upper entry depends on the lower * entries, we must remove entries top - down. */ -static void dequeue_rt_stack (struct task_struct *p) +static void dequeue_rt_stack (struct sched_rt_entity *rt_se) { -struct sched_rt_entity *rt_se, *back =NULL ; +struct sched_rt_entity *back =NULL ; - rt_se = &p->rt;for_each_sched_rt_entity (rt_se) { rt_se->back = back; back = rt_se;
@@ -492,7 +497,26 @@ static void dequeue_rt_stack(struct taskfor (rt_se = back; rt_se; rt_se = rt_se->back) {if (on_rt_rq (rt_se)) -dequeue_rt_entity (rt_se); +__dequeue_rt_entity (rt_se); + } +} + +static void enqueue_rt_entity (struct sched_rt_entity *rt_se) +{ +dequeue_rt_stack (rt_se); +for_each_sched_rt_entity (rt_se) +__enqueue_rt_entity (rt_se); +} + +static void dequeue_rt_entity (struct sched_rt_entity *rt_se) +{ +dequeue_rt_stack (rt_se); + +for_each_sched_rt_entity (rt_se) { +struct rt_rq *rt_rq =group_rt_rq (rt_se); + +if (rt_rq && rt_rq->rt_nr_running) +__enqueue_rt_entity (rt_se); } }
@@ -506,32 +530,15 @@ static void enqueue_task_rt(struct rq *rif (wakeup) rt_se->timeout =0 ; -dequeue_rt_stack (p); - -/* -* enqueue everybody, bottom - up. -*/ -for_each_sched_rt_entity (rt_se) -enqueue_rt_entity (rt_se); +enqueue_rt_entity (rt_se); }static void dequeue_task_rt (struct rq *rq,struct task_struct *p,int sleep) {struct sched_rt_entity *rt_se = &p->rt; -struct rt_rq *rt_rq;update_curr_rt (rq); - -dequeue_rt_stack (p); - -/* -* re-enqueue all non-empty rt_rq entities. -*/ -for_each_sched_rt_entity (rt_se) { - rt_rq =group_rt_rq (rt_se); -if (rt_rq && rt_rq->rt_nr_running) -enqueue_rt_entity (rt_se); - } +dequeue_rt_entity (rt_se); }/*
PATCH 4/10 sched: rt-group: fix RR buglet
In reply to Peter ZijlstraIn tick_task_rt() we first call update_curr_rt() which can dequeue a runqueue due to it running out of runtime, and then we try to requeue it, of it also having exhausted its RR quota. Obviously requeueing something that is no longer on the runqueue will not have the expected result.
--- kernel/sched_rt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
kernel/sched_rt.c +3 -1
@@ -549,8 +549,10 @@ staticvoid requeue_rt_entity (struct rt_rq *rt_rq,struct sched_rt_entity *rt_se) {struct rt_prio_array *array = &rt_rq->active; +struct list_head *queue = array->queue +rt_se_prio (rt_se); -list_move_tail (&rt_se->run_list, array->queue +rt_se_prio (rt_se)); +if (on_rt_rq (rt_se)) +list_move_tail (&rt_se->run_list, queue); }static void requeue_task_rt (struct rq *rq,struct task_struct *p)
PATCH 5/10 sched: rt-bandwidth: fix cpu-hotplug interaction
In reply to Peter ZijlstraRestore the bandwidth to offlining cpus so we don't leak/gain bandwidth.
--- kernel/sched.c | 15 +++++-- kernel/sched_rt.c | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 114 insertions(+), 9 deletions(-)
kernel/sched_rt.c +103 -5
@@ -280,6 +280,9 @@ static int balance_runtime(struct rt_rqcontinue ;spin_lock (&iter->rt_runtime_lock); +if (iter->rt_runtime ==RUNTIME_INF ) + goto next; + diff = iter->rt_runtime - iter->rt_time;if (diff >0 ) {do_div (diff, weight);
@@ -293,12 +296,104 @@ static int balance_runtime(struct rt_rq break; } } +next:spin_unlock (&iter->rt_runtime_lock); }spin_unlock (&rt_b->rt_runtime_lock);return more; } + +static void __disable_runtime (struct rq *rq) +{ +struct root_domain *rd = rq->rd; +struct rt_rq *rt_rq; + +if (unlikely (!scheduler_running)) +return ; + +for_each_leaf_rt_rq (rt_rq, rq) { +struct rt_bandwidth *rt_b =sched_rt_bandwidth (rt_rq); +s64 want; +int i; + +spin_lock (&rt_b->rt_runtime_lock); +spin_lock (&rt_rq->rt_runtime_lock); +if (rt_rq->rt_runtime ==RUNTIME_INF || + rt_rq->rt_runtime == rt_b->rt_runtime) + goto balanced; +spin_unlock (&rt_rq->rt_runtime_lock); + + want = rt_b->rt_runtime - rt_rq->rt_runtime; + +for_each_cpu_mask (i, rd->span) { +struct rt_rq *iter =sched_rt_period_rt_rq (rt_b, i); +s64 diff; + +if (iter == rt_rq) +continue ; + +spin_lock (&iter->rt_runtime_lock); +if (want >0 ) { + diff =min_t (s64, iter->rt_runtime, want); + iter->rt_runtime -= diff; + want -= diff; + }else { + iter->rt_runtime -= want; + want -= want; + } +spin_unlock (&iter->rt_runtime_lock); + +if (!want) +break ; + } + +spin_lock (&rt_rq->rt_runtime_lock); +BUG_ON (want); +balanced: + rt_rq->rt_runtime =RUNTIME_INF ; +spin_unlock (&rt_rq->rt_runtime_lock); +spin_unlock (&rt_b->rt_runtime_lock); + } +} + +static void disable_runtime (struct rq *rq) +{ +unsigned long flags; + +spin_lock_irqsave (&rq->lock, flags); +__disable_runtime (rq); +spin_unlock_irqrestore (&rq->lock, flags); +} + +static void __enable_runtime (struct rq *rq) +{ +struct rt_rq *rt_rq; + +if (unlikely (!scheduler_running)) +return ; + +for_each_leaf_rt_rq (rt_rq, rq) { +struct rt_bandwidth *rt_b =sched_rt_bandwidth (rt_rq); + +spin_lock (&rt_b->rt_runtime_lock); +spin_lock (&rt_rq->rt_runtime_lock); + rt_rq->rt_runtime = rt_b->rt_runtime; + rt_rq->rt_time =0 ; +spin_unlock (&rt_rq->rt_runtime_lock); +spin_unlock (&rt_b->rt_runtime_lock); + } +} + +static void enable_runtime (struct rq *rq) +{ +unsigned long flags; + +spin_lock_irqsave (&rq->lock, flags); +__enable_runtime (rq); +spin_unlock_irqrestore (&rq->lock, flags); +} +#endif static inline int rt_se_prio (struct sched_rt_entity *rt_se)
@@ -328,14 +423,13 @@ static int sched_rt_runtime_exceeded(str#ifdef CONFIG_SMP if (rt_rq->rt_time > runtime) { -int more; -spin_unlock (&rt_rq->rt_runtime_lock); - more =balance_runtime (rt_rq); +balance_runtime (rt_rq);spin_lock (&rt_rq->rt_runtime_lock); -if (more) - runtime =sched_rt_runtime (rt_rq); + runtime =sched_rt_runtime (rt_rq); +if (runtime ==RUNTIME_INF ) +return 0 ; }#endif
@@ -1166,6 +1260,8 @@ static void join_domain_rt(struct rq *rq {if (rq->rt.overloaded)rt_set_overload (rq); + +__enable_runtime (rq); }/* Assumes rq->lock is held */
@@ -1173,6 +1269,8 @@ static void leave_domain_rt(struct rq *r {if (rq->rt.overloaded)rt_clear_overload (rq); + +__disable_runtime (rq); }/*
kernel/sched.c +11 -4
@@ -7461,20 +7461,27 @@ int sched_create_sysfs_power_savings_entstatic int update_sched_domains (struct notifier_block *nfb,unsigned long action,void *hcpu) { +int cpu = (int )(long )hcpu; +switch (action) { -case CPU_UP_PREPARE : -case CPU_UP_PREPARE_FROZEN :case CPU_DOWN_PREPARE :case CPU_DOWN_PREPARE_FROZEN : +disable_runtime (cpu_rq (cpu)); +/* fall-through */ +case CPU_UP_PREPARE : +case CPU_UP_PREPARE_FROZEN :detach_destroy_domains (&cpu_online_map);return NOTIFY_OK ; -case CPU_UP_CANCELED : -case CPU_UP_CANCELED_FROZEN : +case CPU_DOWN_FAILED :case CPU_DOWN_FAILED_FROZEN :case CPU_ONLINE :case CPU_ONLINE_FROZEN : +enable_runtime (cpu_rq (cpu)); +/* fall-through */ +case CPU_UP_CANCELED : +case CPU_UP_CANCELED_FROZEN :case CPU_DEAD :case CPU_DEAD_FROZEN :/*
PATCH 6/10 sched: debug: add some rt debug output
In reply to Peter ZijlstraAlso display rt_rq information in /proc/sched_debug
--- kernel/sched_debug.c | 40 +++++++++++++++++++++++++++++++++++++--- kernel/sched_rt.c | 14 ++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-)
kernel/sched_debug.c +37 -3
@@ -119,9 +119,7 @@ void print_cfs_rq(struct seq_file *m, instruct sched_entity *last;unsigned long flags; -#if !defined(CONFIG_CGROUP_SCHED ) || !defined(CONFIG_USER_SCHED ) -SEQ_printf (m,"\ncfs_rq[%d]:\n" , cpu); -#else +#if defined(CONFIG_CGROUP_SCHED ) && defined(CONFIG_FAIR_GROUP_SCHED )char path[128 ] ="" ;struct cgroup *cgroup =NULL ;struct task_group *tg = cfs_rq->tg;
@@ -133,6 +131,8 @@ void print_cfs_rq(struct seq_file *m, incgroup_path (cgroup, path,sizeof (path));SEQ_printf (m,"\ncfs_rq[%d]:%s\n" , cpu, path); +#else +SEQ_printf (m,"\ncfs_rq[%d]:\n" , cpu);#endif SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
@@ -169,6 +169,39 @@ void print_cfs_rq(struct seq_file *m, in cfs_rq->nr_spread_over); } +void print_rt_rq (struct seq_file *m,int cpu,struct rt_rq *rt_rq) +{ +#if defined(CONFIG_CGROUP_SCHED ) && defined(CONFIG_RT_GROUP_SCHED ) +char path[128 ] ="" ; +struct cgroup *cgroup =NULL ; +struct task_group *tg = rt_rq->tg; + +if (tg) + cgroup = tg->css.cgroup; + +if (cgroup) +cgroup_path (cgroup, path,sizeof (path)); + +SEQ_printf (m,"\nrt_rq[%d]:%s\n" , cpu, path); +#else +SEQ_printf (m,"\nrt_rq[%d]:\n" , cpu); +#endif + + +#define P (x) \ + SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x)) +#define PN (x) \ + SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x)) + +P (rt_nr_running); +P (rt_throttled); +PN (rt_time); +PN (rt_runtime); + +#undef PN +#undef P +} +static void print_cpu (struct seq_file *m,int cpu) {struct rq *rq = &per_cpu (runqueues, cpu);
@@ -208,6 +241,7 @@ static void print_cpu(struct seq_file *m#undef PNprint_cfs_stats (m, cpu); +print_rt_stats (m, cpu);print_rq (m, rq, cpu); }
kernel/sched_rt.c +14 -0
@@ -1447,3 +1447,17 @@ static const struct sched_class rt_sched .prio_changed = prio_changed_rt, .switched_to = switched_to_rt, }; + +#ifdef CONFIG_SCHED_DEBUG +extern void print_rt_rq (struct seq_file *m,int cpu,struct rt_rq *rt_rq); + +static void print_rt_stats (struct seq_file *m,int cpu) +{ +struct rt_rq *rt_rq; + +rcu_read_lock (); +for_each_leaf_rt_rq (rt_rq,cpu_rq (cpu)) +print_rt_rq (m, cpu, rt_rq); +rcu_read_unlock (); +} +#endif
PATCH 7/10 sched: rt: fix SMP bandwidth balancing for throttled groups
In reply to Peter ZijlstraWe didn't balance the runtime when throttled, this can cause large wakeup latencies. Suppose a task is migrated to another cpu right before the group quota runs out - its likely that the previuos cpu had a large amount of the group runtime, whereas the new cpu would be almost depleted (due to it being handed the other cpu).
Now we exceed the runtime and get throttled - the period rollover tick will subtract the cpu quota from the runtime and check if we're below quota. However with this cpu having a very small portion of the runtime it will not refresh as fast as it should.
Therefore, also rebalance the runtime when we're throttled.
--- kernel/sched_rt.c | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-)
kernel/sched_rt.c +29 -12
@@ -222,6 +222,28 @@ static inline struct rt_bandwidth *sched#endif +#ifdef CONFIG_SMP +static int do_balance_runtime (struct rt_rq *rt_rq); + +static int balance_runtime (struct rt_rq *rt_rq) +{ +int more =0 ; + +if (rt_rq->rt_time > rt_rq->rt_runtime) { +spin_unlock (&rt_rq->rt_runtime_lock); + more =do_balance_runtime (rt_rq); +spin_lock (&rt_rq->rt_runtime_lock); + } + +return more; +} +#else +static inline int balance_runtime (struct rt_rq *rt_rq) +{ +return 0 ; +} +#endif +static int do_sched_rt_period_timer (struct rt_bandwidth *rt_b,int overrun) {int i, idle =1 ;
@@ -241,6 +263,8 @@ static int do_sched_rt_period_timer(struu64 runtime;spin_lock (&rt_rq->rt_runtime_lock); +if (rt_rq->rt_throttled) +balance_runtime (rt_rq); runtime = rt_rq->rt_runtime; rt_rq->rt_time -=min (rt_rq->rt_time, overrun*runtime);if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
@@ -261,7 +285,7 @@ static int do_sched_rt_period_timer(stru }#ifdef CONFIG_SMP -static int balance_runtime (struct rt_rq *rt_rq) +static int do_balance_runtime (struct rt_rq *rt_rq) {struct rt_bandwidth *rt_b =sched_rt_bandwidth (rt_rq);struct root_domain *rd =cpu_rq (smp_processor_id ())->rd;
@@ -422,17 +446,10 @@ static int sched_rt_runtime_exceeded(strif (sched_rt_runtime (rt_rq) >=sched_rt_period (rt_rq))return 0 ; -#ifdef CONFIG_SMP -if (rt_rq->rt_time > runtime) { -spin_unlock (&rt_rq->rt_runtime_lock); -balance_runtime (rt_rq); -spin_lock (&rt_rq->rt_runtime_lock); - - runtime =sched_rt_runtime (rt_rq); -if (runtime ==RUNTIME_INF ) -return 0 ; - } -#endif +balance_runtime (rt_rq); + runtime =sched_rt_runtime (rt_rq); +if (runtime ==RUNTIME_INF ) +return 0 ;if (rt_rq->rt_time > runtime) { rt_rq->rt_throttled =1 ;
PATCH 8/10 sched: rt: move some code around
In reply to Peter ZijlstraShuffle some code around to get rid of some #ifdef
--- kernel/sched_rt.c | 119 +++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 62 deletions(-)
kernel/sched_rt.c +57 -62
@@ -223,68 +223,6 @@ static inline struct rt_bandwidth *sched#endif #ifdef CONFIG_SMP -static int do_balance_runtime (struct rt_rq *rt_rq); - -static int balance_runtime (struct rt_rq *rt_rq) -{ -int more =0 ; - -if (rt_rq->rt_time > rt_rq->rt_runtime) { -spin_unlock (&rt_rq->rt_runtime_lock); - more =do_balance_runtime (rt_rq); -spin_lock (&rt_rq->rt_runtime_lock); - } - -return more; -} -#else -static inline int balance_runtime (struct rt_rq *rt_rq) -{ -return 0 ; -} -#endif - -static int do_sched_rt_period_timer (struct rt_bandwidth *rt_b,int overrun) -{ -int i, idle =1 ; -cpumask_t span; - -if (rt_b->rt_runtime ==RUNTIME_INF ) -return 1 ; - - span =sched_rt_period_mask (); -for_each_cpu_mask (i, span) { -int enqueue =0 ; -struct rt_rq *rt_rq =sched_rt_period_rt_rq (rt_b, i); -struct rq *rq =rq_of_rt_rq (rt_rq); - -spin_lock (&rq->lock); -if (rt_rq->rt_time) { -u64 runtime; - -spin_lock (&rt_rq->rt_runtime_lock); -if (rt_rq->rt_throttled) -balance_runtime (rt_rq); - runtime = rt_rq->rt_runtime; - rt_rq->rt_time -=min (rt_rq->rt_time, overrun*runtime); -if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { - rt_rq->rt_throttled =0 ; - enqueue =1 ; - } -if (rt_rq->rt_time || rt_rq->rt_nr_running) - idle =0 ; -spin_unlock (&rt_rq->rt_runtime_lock); - } - -if (enqueue) -sched_rt_rq_enqueue (rt_rq); -spin_unlock (&rq->lock); - } - -return idle; -} - -#ifdef CONFIG_SMP static int do_balance_runtime (struct rt_rq *rt_rq) {struct rt_bandwidth *rt_b =sched_rt_bandwidth (rt_rq);
@@ -419,8 +357,65 @@ static void enable_runtime(struct rq *rqspin_unlock_irqrestore (&rq->lock, flags); } +static int balance_runtime (struct rt_rq *rt_rq) +{ +int more =0 ; + +if (rt_rq->rt_time > rt_rq->rt_runtime) { +spin_unlock (&rt_rq->rt_runtime_lock); + more =do_balance_runtime (rt_rq); +spin_lock (&rt_rq->rt_runtime_lock); + } + +return more; +} +#else +static inline int balance_runtime (struct rt_rq *rt_rq) +{ +return 0 ; +}#endif +static int do_sched_rt_period_timer (struct rt_bandwidth *rt_b,int overrun) +{ +int i, idle =1 ; +cpumask_t span; + +if (rt_b->rt_runtime ==RUNTIME_INF ) +return 1 ; + + span =sched_rt_period_mask (); +for_each_cpu_mask (i, span) { +int enqueue =0 ; +struct rt_rq *rt_rq =sched_rt_period_rt_rq (rt_b, i); +struct rq *rq =rq_of_rt_rq (rt_rq); + +spin_lock (&rq->lock); +if (rt_rq->rt_time) { +u64 runtime; + +spin_lock (&rt_rq->rt_runtime_lock); +if (rt_rq->rt_throttled) +balance_runtime (rt_rq); + runtime = rt_rq->rt_runtime; + rt_rq->rt_time -=min (rt_rq->rt_time, overrun*runtime); +if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { + rt_rq->rt_throttled =0 ; + enqueue =1 ; + } +if (rt_rq->rt_time || rt_rq->rt_nr_running) + idle =0 ; +spin_unlock (&rt_rq->rt_runtime_lock); + } + +if (enqueue) +sched_rt_rq_enqueue (rt_rq); +spin_unlock (&rq->lock); + } + +return idle; +} +static inline int rt_se_prio (struct sched_rt_entity *rt_se) {#ifdef CONFIG_RT_GROUP_SCHED
PATCH 9/10 sched: rt: fix the bandwidth contraint computations
In reply to Peter Zijlstra - allow a subgroup to use all of its parent's bandwidth
- contrain the global bandwidth to not be lower than the root group's
--- kernel/sched.c | 11 ++++++++--- kernel/sched_rt.c | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-)
kernel/sched.c +8 -3
@@ -8354,7 +8354,7 @@ static unsigned long to_ratio(u64 period#ifdef CONFIG_CGROUP_SCHED static int __rt_schedulable (struct task_group *tg,u64 period,u64 runtime) { -struct task_group *tgi, *parent = tg ? tg->parent :NULL ; +struct task_group *tgi, *parent = tg->parent;unsigned long total =0 ;if (!parent) {
@@ -8378,7 +8378,7 @@ static int __rt_schedulable(struct task_ }rcu_read_unlock (); -return total +to_ratio (period, runtime) < +return total +to_ratio (period, runtime) <=to_ratio (ktime_to_ns (parent->rt_bandwidth.rt_period), parent->rt_bandwidth.rt_runtime); }
@@ -8495,10 +8495,15 @@ long sched_group_rt_period(struct task_gstatic int sched_rt_global_constraints (void ) { +struct task_group *tg = &root_task_group; +u64 rt_runtime, rt_period;int ret =0 ; + rt_period =ktime_to_ns (tg->rt_bandwidth.rt_period); + rt_runtime = tg->rt_bandwidth.rt_runtime; +mutex_lock (&rt_constraints_mutex); -if (!__rt_schedulable (NULL ,1 ,0 )) +if (!__rt_schedulable (tg, rt_period, rt_runtime)) ret = -EINVAL ;mutex_unlock (&rt_constraints_mutex);
PATCH 10/10 sched: rt: dont stop the period timer when there are tasks wanting to run
In reply to Peter ZijlstraWhen a runnable group failed to get any runtime within a period it would be found idle and the period timer would be stopped.
So if the group ever gets throttled, it will never wake up again.
--- kernel/sched_rt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/sched_rt.c +2 -1
@@ -408,7 +408,8 @@ static int do_sched_rt_period_timer(struif (rt_rq->rt_time || rt_rq->rt_nr_running) idle =0 ;spin_unlock (&rt_rq->rt_runtime_lock); - } + }else if (rt_rq->rt_nr_running) + idle =0 ;if (enqueue)sched_rt_rq_enqueue (rt_rq);
Re: [PATCH 10/10] sched: rt: dont stop the period timer when there are tasks wanting to run
Quoted 7 lines · Peter Zijlstra
When a runnable group failed to get any runtime within a period it would be found idle and the period timer would be stopped.
So if the group ever gets throttled, it will never wake up again.
Signed-off-by Peter Zijlstra <[email protected]>Reported-by "Daniel K." <[email protected]>
I think this should go into .26 as well, due to the non-intuitiveness of the knobs that cause this if they are fiddled with in the wrong way.
Daniel K.
Quoted 19 lines
--- kernel/sched_rt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Index:
linux-2.6/kernel/sched_rt.c
===================================================================kernel/sched_rt.c +2
-1@@ -408,7 +408,8 @@ static int do_sched_rt_period_timer(struif (rt_rq->rt_time || rt_rq->rt_nr_running) idle =0 ;spin_unlock (&rt_rq->rt_runtime_lock); - } + }else if (rt_rq->rt_nr_running) + idle =0 ;if (enqueue)sched_rt_rq_enqueue (rt_rq);
Re: [PATCH 10/10] sched: rt: dont stop the period timer when there are tasks wanting to run
Quoted 4 lines · Daniel K.
Quoted 3 lines · Peter Zijlstra
When a runnable group failed to get any runtime within a period it would be found idle and the period timer would be stopped.
Quoted 8 lines
Quoted 4 lines
So if the group ever gets throttled, it will never wake up again.
Signed-off-by Peter Zijlstra <[email protected]>Reported-by "Daniel K." <[email protected]>Tested-by Daniel K. <[email protected]>I think this should go into .26 as well, due to the non-intuitiveness of the knobs that cause this if they are fiddled with in the wrong way.
Sounds reasonable, Ingo?
Quoted 20 lines
Quoted 19 lines
--- kernel/sched_rt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)Index:
linux-2.6/kernel/sched_rt.c
===================================================================kernel/sched_rt.c +2
-1@@ -408,7 +408,8 @@ static int do_sched_rt_period_timer(struif (rt_rq->rt_time || rt_rq->rt_nr_running) idle =0 ;spin_unlock (&rt_rq->rt_runtime_lock); - } + }else if (rt_rq->rt_nr_running) + idle =0 ;if (enqueue)sched_rt_rq_enqueue (rt_rq);
Re: [PATCH 10/10] sched: rt: dont stop the period timer when there are tasks wanting to run
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Daniel K.
Quoted 5 lines · Peter Zijlstra
When a runnable group failed to get any runtime within a period it would be found idle and the period timer would be stopped.
So if the group ever gets throttled, it will never wake up again.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
Signed-off-by Peter Zijlstra <[email protected]>Reported-by "Daniel K." <[email protected]>Tested-by Daniel K. <[email protected]>I think this should go into .26 as well, due to the non-intuitiveness of the knobs that cause this if they are fiddled with in the wrong way.
Sounds reasonable, Ingo?
agreed - i've put it into tip/sched/urgent as well.
Ingo
Ingo Molnar
AppliedQuoted 4 lines · Peter Zijlstra
my current queue of fixes
1-4 are pushed into sched-urgent and should be included in .26 the rest are less critical and should be queued for .27
applied to tip/sched/devel - thanks Peter.
Ingo