scheduler tree updates for v3.20
Subsystems (2)
Linus,
Please pull the latest sched-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched-core-for-linus
# HEAD: 139b6fd26d85a65c4e0d2795b87b94f9505e5943 sched/Documentation: Remove unneeded word
The main scheduler changes in this cycle were:
- various sched/deadline fixes and enhancements
- rescheduling latency fixes/cleanups
- rework the rq->clock code to be more consistent and more robust.
- minor micro-optimizations
- ->avg.decay_count fixes
- add a stack overflow check to might_sleep()
- idle-poll handler fix, possibly resulting in power savings
- misc smaller updates and fixes
Thanks,
Ingo
------------------>
Eric Sandeen (1):
sched/debug: Check for stack overflow in ___might_sleep()
Frederic Weisbecker (2):
sched: Fix missing preemption opportunity
sched: Pull resched loop to __schedule() callers
Johan Hedberg (1):
sched/wait: Introduce wait_on_bit_timeout()
Kirill Tkhai (1):
sched/fair: Fix sched_entity::avg::decay_count initialization
Peter Zijlstra (4):
sched/core: Validate rq_clock*() serialization sched/core: Rework rq->clock update skips sched/debug: Print rq->clock_task sched/deadline: Fix stale yield state
Preeti U Murthy (1):
sched/idle: Add missing checks to the exit condition of cpu_idle_poll()
Sharon Dvir (1):
sched/Documentation: Remove unneeded word
Tetsuo Handa (1):
sched/debug: Fix potential call to __ffs(0) in sched_show_task()
Tim Chen (1):
sched/rt: Reduce rq lock contention by eliminating locking of non-feasible target
Wanpeng Li (3):
sched/deadline: Fix hrtick for a non-leftmost task sched/deadline: Avoid pointless __setscheduler()
sched: Fix hrtick_start() on UP
Xunlei Pang (3):
sched/fair: Fix the dealing with decay_count in __synchronize_entity_decay() sched/deadline: Modify cpudl::free_cpus to reflect rd->online sched/deadline: Remove cpu_active_mask from cpudl_find()
Yao Dongdong (1):
sched/core: Remove check of p->sched_class
include/linux/wait.h | 26 +++++++++++ kernel/locking/mutex.c | 2 +- kernel/sched/core.c | 107 +++++++++++++++++++++++++++++---------------- kernel/sched/cpudeadline.c | 27 ++++++++++-- kernel/sched/cpudeadline.h | 2 + kernel/sched/deadline.c | 51 +++++++++++---------- kernel/sched/debug.c | 1 + kernel/sched/fair.c | 7 ++- kernel/sched/idle.c | 3 +- kernel/sched/rt.c | 26 +++++++++-- kernel/sched/sched.h | 22 +++++++++- 11 files changed, 197 insertions(+), 77 deletions(-)
include/linux/wait.h +26 -0
@@ -990,6 +990,32 @@ wait_on_bit_io(void *word, int bit, unsigned mode) }/** +* wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses +* @word: the word being waited on, a kernel virtual address +* @bit: the bit of the word being waited on +* @mode: the task state to sleep in +* @timeout: timeout, in jiffies +* +* Use the standard hashed waitqueue table to wait for a bit +* to be cleared. This is similar to wait_on_bit(), except also takes a +* timeout parameter. +* +* Returned value will be zero if the bit was cleared before the +* @timeout elapsed, or non-zero if the @timeout elapsed or process +* received a signal and the mode permitted wakeup on that signal. +*/ +static inline int +wait_on_bit_timeout (void *word,int bit,unsigned mode,unsigned long timeout) +{ +might_sleep (); +if (!test_bit (bit, word)) +return 0 ; +return out_of_line_wait_on_bit_timeout (word, bit, + bit_wait_timeout, + mode, timeout); +} + +/** * wait_on_bit_action - wait for a bit to be cleared * @word: the word being waited on, a kernel virtual address * @bit: the bit of the word being waited on
kernel/locking/mutex.c +1 -1
@@ -81,7 +81,7 @@ __visible void __sched __mutex_lock_slowpath(atomic_t *lock_count);* The mutex must later on be released by the same task that * acquired it. Recursive locking is not allowed. The task * may not exit without first unlocking the mutex. Also, kernel -* memory where the mutex resides mutex must not be freed with +* memory where the mutex resides must not be freed with * the mutex still locked. The mutex must first be initialized * (or statically defined) before it can be locked. memset()-ing * the mutex to 0 is not allowed.
kernel/sched/core.c +69 -38
248 lines not on this page · raw message
kernel/sched/cpudeadline.c +23 -4
@@ -107,7 +107,8 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,int best_cpu =-1 ;const struct sched_dl_entity *dl_se = &p->dl; -if (later_mask &&cpumask_and (later_mask, later_mask, cp->free_cpus)) { +if (later_mask && +cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { best_cpu =cpumask_any (later_mask); goto out; } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) &&
@@ -186,6 +187,26 @@ void cpudl_set(struct cpudl *cp, int cpu, u64 dl, int is_valid) }/* +* cpudl_set_freecpu - Set the cpudl.free_cpus +* @cp: the cpudl max-heap context +* @cpu: rd attached cpu +*/ +void cpudl_set_freecpu (struct cpudl *cp,int cpu) +{ +cpumask_set_cpu (cpu, cp->free_cpus); +} + +/* +* cpudl_clear_freecpu - Clear the cpudl.free_cpus +* @cp: the cpudl max-heap context +* @cpu: rd attached cpu +*/ +void cpudl_clear_freecpu (struct cpudl *cp,int cpu) +{ +cpumask_clear_cpu (cpu, cp->free_cpus); +} + +/* * cpudl_init - initialize the cpudl structure * @cp: the cpudl max-heap context */
@@ -203,7 +224,7 @@ int cpudl_init(struct cpudl *cp)if (!cp->elements)return -ENOMEM ; -if (!alloc_cpumask_var (&cp->free_cpus,GFP_KERNEL )) { +if (!zalloc_cpumask_var (&cp->free_cpus,GFP_KERNEL )) {kfree (cp->elements);return -ENOMEM ; }
@@ -211,8 +232,6 @@ int cpudl_init(struct cpudl *cp)for_each_possible_cpu (i) cp->elements[i].idx =IDX_INVALID ; -cpumask_setall (cp->free_cpus); -return 0 ; }
kernel/sched/cpudeadline.h +2 -0
@@ -24,6 +24,8 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, struct cpumask *later_mask);void cpudl_set (struct cpudl *cp,int cpu,u64 dl,int is_valid);int cpudl_init (struct cpudl *cp); +void cpudl_set_freecpu (struct cpudl *cp,int cpu); +void cpudl_clear_freecpu (struct cpudl *cp,int cpu);void cpudl_cleanup (struct cpudl *cp);#endif /* CONFIG_SMP */
kernel/sched/deadline.c +28 -23
102 lines not on this page · raw message
kernel/sched/debug.c +1 -0
@@ -305,6 +305,7 @@ do { \PN (next_balance);SEQ_printf (m," .%-30s: %ld\n" ,"curr->pid" , (long )(task_pid_nr (rq->curr)));PN (clock); +PN (clock_task);P (cpu_load[0 ]);P (cpu_load[1 ]);P (cpu_load[2 ]);
kernel/sched/fair.c +3 -4
@@ -676,7 +676,6 @@ void init_task_runnable_average(struct task_struct *p) {u32 slice; - p->se.avg.decay_count =0 ; slice =sched_slice (task_cfs_rq (p), &p->se) >>10 ; p->se.avg.runnable_avg_sum = slice; p->se.avg.runnable_avg_period = slice;
@@ -2574,11 +2573,11 @@ static inline u64 __synchronize_entity_decay(struct sched_entity *se)u64 decays =atomic64_read (&cfs_rq->decay_counter); decays -= se->avg.decay_count; + se->avg.decay_count =0 ;if (!decays)return 0 ; se->avg.load_avg_contrib =decay_load (se->avg.load_avg_contrib, decays); - se->avg.decay_count =0 ;return decays; }
@@ -5157,7 +5156,7 @@ static void yield_task_fair(struct rq *rq)* so we don't do microscopic update in schedule() * and double the fastpath cost. */ - rq->skip_clock_update =1 ; +rq_clock_skip_update (rq, true); }set_skip_buddy (se);
@@ -5949,8 +5948,8 @@ static unsigned long scale_rt_capacity(int cpu)*/ age_stamp =ACCESS_ONCE (rq->age_stamp); avg =ACCESS_ONCE (rq->rt_avg); + delta =__rq_clock_broken (rq) - age_stamp; - delta =rq_clock (rq) - age_stamp;if (unlikely (delta <0 )) delta =0 ;
kernel/sched/idle.c +2 -1
@@ -47,7 +47,8 @@ static inline int cpu_idle_poll(void)rcu_idle_enter ();trace_cpu_idle_rcuidle (0 ,smp_processor_id ());local_irq_enable (); -while (!tif_need_resched ()) +while (!tif_need_resched () && + (cpu_idle_force_poll ||tick_check_broadcast_expired ()))cpu_relax ();trace_cpu_idle_rcuidle (PWR_EVENT_EXIT ,smp_processor_id ());rcu_idle_exit ();
kernel/sched/rt.c +22 -4
@@ -831,11 +831,14 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) enqueue =1 ;/* -* Force a clock update if the CPU was idle, -* lest wakeup -> unthrottle time accumulate. +* When we're idle and a woken (rt) task is +* throttled check_preempt_curr() will set +* skip_update and the time between the wakeup +* and this unthrottle will get accounted as +* 'runtime'. */ if (rt_rq->rt_nr_running && rq->curr == rq->idle) - rq->skip_clock_update =-1 ; +rq_clock_skip_update (rq, false); }if (rt_rq->rt_time || rt_rq->rt_nr_running) idle =0 ;
@@ -1337,7 +1340,12 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) curr->prio <= p->prio)) {int target =find_lowest_rq (p); -if (target !=-1 ) +/* +* Don't bother moving it if the destination CPU is +* not running a lower priority task. +*/ +if (target !=-1 && + p->prio <cpu_rq (target)->rt.highest_prio.curr) cpu = target; }rcu_read_unlock ();
@@ -1614,6 +1622,16 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) lowest_rq =cpu_rq (cpu); +if (lowest_rq->rt.highest_prio.curr <= task->prio) { +/* +* Target rq has tasks of equal or higher priority, +* retrying does not release any lock and is unlikely +* to yield a different result. +*/ + lowest_rq =NULL ; +break ; + } +/* if the prio of this runqueue changed, try again */ if (double_lock_balance (rq, lowest_rq)) {/*
kernel/sched/sched.h +20 -2
@@ -558,8 +558,6 @@ struct rq {#ifdef CONFIG_NO_HZ_FULL unsigned long last_sched_tick;#endif -int skip_clock_update; -/* capture load from *all* tasks on this cpu: */ struct load_weight load;unsigned long nr_load_updates;
@@ -588,6 +586,7 @@ struct rq {unsigned long next_balance;struct mm_struct *prev_mm; +unsigned int clock_skip_update;u64 clock;u64 clock_task;
@@ -687,16 +686,35 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);#define cpu_curr (cpu) (cpu_rq(cpu)->curr)#define raw_rq () raw_cpu_ptr(&runqueues) +static inline u64 __rq_clock_broken (struct rq *rq) +{ +return ACCESS_ONCE (rq->clock); +} +static inline u64 rq_clock (struct rq *rq) { +lockdep_assert_held (&rq->lock);return rq->clock; }static inline u64 rq_clock_task (struct rq *rq) { +lockdep_assert_held (&rq->lock);return rq->clock_task; } +#define RQCF_REQ_SKIP 0x01 +#define RQCF_ACT_SKIP 0x02 + +static inline void rq_clock_skip_update (struct rq *rq,bool skip) +{ +lockdep_assert_held (&rq->lock); +if (skip) + rq->clock_skip_update |=RQCF_REQ_SKIP ; +else + rq->clock_skip_update &= ~RQCF_REQ_SKIP ; +} +#ifdef CONFIG_NUMA enum numa_topology_type { NUMA_DIRECT,