sched/core: Optimize the process of picking the max prio task for the core
PATCH v2 1/1 sched/core: Optimize the process of picking the max prio task for the core
When we pick the max prio task for the core in the case of sched_core_enabled(), if there's a task with a higher prio sched class in the runqueue of a SMT, it's not necessary for other SMTs to traverse lower prio sched classes. So we can change the traversal
order: for each sched class, if there exists a max prio task among the core, pick it and break the loop.
To compare the prio of the tasks with the same sched class, we introduce sched_class::prio_less().
Signed-off-by Cruz Zhao <[email protected]>
Reported-by kernel test robot <[email protected]>
---
Changes since v1:
Fixes the compile error.
--- kernel/sched/core.c | 33 ++++++++++++++++++++++----------- kernel/sched/deadline.c | 11 +++++++++++ kernel/sched/fair.c | 5 ++++- kernel/sched/idle.c | 10 ++++++++++ kernel/sched/rt.c | 10 ++++++++++ kernel/sched/sched.h | 5 +++-- kernel/sched/stop_task.c | 10 ++++++++++ 7 files changed, 70 insertions(+), 14 deletions(-)
kernel/sched/core.c +22 -11
@@ -189,7 +189,7 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b, boolreturn !dl_time_before (a->dl.deadline, b->dl.deadline);if (pa ==MAX_RT_PRIO +MAX_NICE )/* fair */ -return cfs_prio_less (a, b, in_fi); +return fair_sched_class.prio_less (a, b, in_fi);return false; }
@@ -5886,6 +5886,7 @@ static inline struct task_struct *pick_task(struct rq *rq)int i, cpu, occ =0 ;struct rq *rq_i;bool need_sync; +struct sched_class *class;if (!sched_core_enabled (rq))return __pick_next_task (rq, prev, rf);
@@ -5978,12 +5979,6 @@ static inline struct task_struct *pick_task(struct rq *rq) } } -/* -* For each thread: do the regular task pick and find the max prio task -* amongst them. -* -* Tie-break prio towards the current CPU -*/ for_each_cpu_wrap (i, smt_mask, cpu) { rq_i =cpu_rq (i);
@@ -5994,12 +5989,28 @@ static inline struct task_struct *pick_task(struct rq *rq)*/ if (i != cpu && (rq_i != rq->core || !core_clock_updated))update_rq_clock (rq_i); + } - p = rq_i->core_pick =pick_task (rq_i); -if (!max ||prio_less (max, p, fi_before)) - max = p; +/* +* For each thread: do the regular task pick and find the max prio task +* amongst them. +* +* Tie-break prio towards the current CPU +*/ +for_each_class (class) { +for_each_cpu_wrap (i, smt_mask, cpu) { + rq_i =cpu_rq (i); + p = rq_i->core_pick = class->pick_task (rq_i); +if (!max || (p && class->prio_less (max, p, fi_before))) + max = p; + } +if (max) +break ; } +if (!max) +BUG (); + cookie = rq->core->core_cookie = max->core_cookie;/*
@@ -6010,7 +6021,7 @@ static inline struct task_struct *pick_task(struct rq *rq) rq_i =cpu_rq (i); p = rq_i->core_pick; -if (!cookie_equals (p, cookie)) { +if (!p || !cookie_equals (p, cookie)) { p =NULL ;if (cookie) p =sched_core_find (rq_i, cookie);
kernel/sched/deadline.c +11 -0
@@ -1942,6 +1942,7 @@ static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf) return sched_stop_runnable(rq) || sched_dl_runnable(rq); } +#endif /* CONFIG_SMP */ /*
@@ -2054,6 +2055,13 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p)enqueue_pushable_dl_task (rq, p); } +#ifdef CONFIG_SCHED_CORE +static bool prio_less_dl (struct task_struct *a,struct task_struct *b,bool in_fi) +{ +return !dl_time_before (a->dl.deadline, b->dl.deadline); +} +#endif +/* * scheduler tick hitting a task of our scheduling class. *
@@ -2704,6 +2712,9 @@ static void prio_changed_dl(struct rq *rq, struct task_struct *p, .pick_next_task = pick_next_task_dl, .put_prev_task = put_prev_task_dl, .set_next_task = set_next_task_dl, +#ifdef CONFIG_SCHED_CORE + .prio_less = prio_less_dl, +#endif #ifdef CONFIG_SMP .balance = balance_dl,
kernel/sched/fair.c +4 -1
@@ -11516,7 +11516,7 @@ void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)se_fi_update (se, rq->core->core_forceidle_seq, in_fi); } -bool cfs_prio_less (struct task_struct *a,struct task_struct *b,bool in_fi) +static bool prio_less_fair (struct task_struct *a,struct task_struct *b,bool in_fi) {struct rq *rq =task_rq (a);struct sched_entity *sea = &a->se;
@@ -12159,6 +12159,9 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task .pick_next_task = __pick_next_task_fair, .put_prev_task = put_prev_task_fair, .set_next_task = set_next_task_fair, +#ifdef CONFIG_SCHED_CORE + .prio_less = prio_less_fair, +#endif #ifdef CONFIG_SMP .balance = balance_fair,
kernel/sched/idle.c +10 -0
@@ -436,6 +436,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool firschedstat_inc (rq->sched_goidle); } +#ifdef CONFIG_SCHED_CORE +static bool prio_less_idle (struct task_struct *a,struct task_struct *b,bool in_fi) +{ +return false; +} +#endif +#ifdef CONFIG_SMP static struct task_struct *pick_task_idle (struct rq *rq) {
@@ -507,6 +514,9 @@ static void update_curr_idle(struct rq *rq) .pick_next_task = pick_next_task_idle, .put_prev_task = put_prev_task_idle, .set_next_task = set_next_task_idle, +#ifdef CONFIG_SCHED_CORE + .prio_less = prio_less_idle, +#endif #ifdef CONFIG_SMP .balance = balance_idle,
kernel/sched/rt.c +10 -0
@@ -1838,6 +1838,13 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)enqueue_pushable_task (rq, p); } +static bool prio_less_rt (struct task_struct *a,struct task_struct *b,bool in_fi) +{ +int pa =rt_prio (a->prio), pb =rt_prio (b->prio); + +return -pa < -pb; +} +#ifdef CONFIG_SMP /* Only try algorithms three times */
@@ -2685,6 +2692,9 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) .pick_next_task = pick_next_task_rt, .put_prev_task = put_prev_task_rt, .set_next_task = set_next_task_rt, +#ifdef CONFIG_SCHED_CORE + .prio_less = prio_less_rt, +#endif #ifdef CONFIG_SMP .balance = balance_rt,
kernel/sched/sched.h +3 -2
@@ -1218,8 +1218,6 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)return &rq->__lock; } -bool cfs_prio_less (struct task_struct *a,struct task_struct *b,bool fi); -/* * Helpers to check if the CPU's core cookie matches with the task's cookie * when core scheduling is enabled.
@@ -2155,6 +2153,9 @@ struct sched_class {void (*put_prev_task)(struct rq *rq,struct task_struct *p);void (*set_next_task)(struct rq *rq,struct task_struct *p,bool first); +#ifdef CONFIG_SCHED_CORE +bool (*prio_less)(struct task_struct *a,struct task_struct *b,bool in_fi); +#endif #ifdef CONFIG_SMP int (*balance)(struct rq *rq,struct task_struct *prev,struct rq_flags *rf);
kernel/sched/stop_task.c +10 -0
@@ -84,6 +84,13 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)update_current_exec_runtime (curr, now, delta_exec); } +#ifdef CONFIG_SCHED_CORE +static bool prio_less_stop (struct task_struct *a,struct task_struct *b,bool in_fi) +{ +return false; +} +#endif +/* * scheduler tick hitting a task of our scheduling class. *
@@ -125,6 +132,9 @@ static void update_curr_stop(struct rq *rq) .pick_next_task = pick_next_task_stop, .put_prev_task = put_prev_task_stop, .set_next_task = set_next_task_stop, +#ifdef CONFIG_SCHED_CORE + .prio_less = prio_less_stop, +#endif #ifdef CONFIG_SMP .balance = balance_stop,
Signature · 1 line
1.8.3.1