sched/deadline: Fix wrong cpudl_find() in check_preempt_equal_dl()
PATCH RESEND 1/2 sched/deadline: Fix wrong cpudl_find() in check_preempt_equal_dl()
From: Xunlei Pang <[email protected]>
In check_preempt_equal_dl(), cpudl_find() is called with a NULL later_mask, thus cpudl_find() here doesn't check cpudl::free_cpus at all.
This patch takles this issue by always passing a non-NULL later_mask to cpudl_find(), thereby fixing this issue.
--- kernel/sched/cpudeadline.c | 8 +++----- kernel/sched/deadline.c | 15 +++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-)
kernel/sched/cpudeadline.c +3 -5
@@ -97,7 +97,7 @@ static inline int cpudl_maximum(struct cpudl *cp)* cpudl_find - find the best (later-dl) CPU in the system * @cp: the cpudl max-heap context * @p: the task -* @later_mask: a mask to fill in with the selected CPUs (or NULL) +* @later_mask: a mask to fill in with the selected CPUs (not NULL) * * Returns: int - best CPU (heap maximum if suitable) */
@@ -107,15 +107,13 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,int best_cpu =-1 ;const struct sched_dl_entity *dl_se = &p->dl; -if (later_mask && -cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { +if (cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { best_cpu =cpumask_any (later_mask); goto out; }else if (cpumask_test_cpu (cpudl_maximum (cp), &p->cpus_allowed) &&dl_time_before (dl_se->deadline, cp->elements[0 ].dl)) { best_cpu =cpudl_maximum (cp); -if (later_mask) -cpumask_set_cpu (best_cpu, later_mask); +cpumask_set_cpu (best_cpu, later_mask); } out:
kernel/sched/deadline.c +11 -4
@@ -964,14 +964,23 @@ out:return cpu; } +static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); +static void check_preempt_equal_dl (struct rq *rq,struct task_struct *p) { +struct cpumask *later_mask = +this_cpu_cpumask_var_ptr (local_cpu_mask_dl); + +/* Make sure the mask is initialized first */ +if (unlikely (!later_mask)) +return ; +/* * Current can't be migrated, useless to reschedule, * let's hope p can move out. */ if (rq->curr->nr_cpus_allowed ==1 || -cpudl_find (&rq->rd->cpudl, rq->curr,NULL ) ==-1 ) +cpudl_find (&rq->rd->cpudl, rq->curr, later_mask) ==-1 )return ;/*
@@ -979,7 +988,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)* see if it is pushed or pulled somewhere else. */ if (p->nr_cpus_allowed !=1 && -cpudl_find (&rq->rd->cpudl, p,NULL ) !=-1 ) +cpudl_find (&rq->rd->cpudl, p, later_mask) !=-1 )return ;resched_curr (rq);
@@ -1173,8 +1182,6 @@ next_node:return NULL ; } -static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); -static int find_later_rq (struct task_struct *task) {struct sched_domain *sd;
Signature · 1 line
1.9.1
PATCH RESEND 2/2 sched/rt: Consider deadline tasks in cpupri_find()
From: Xunlei Pang <[email protected]>
Currently, RT global scheduling doesn't factor deadline tasks, this may cause some problems.
See a case below:
On a 3 CPU system, CPU0 has one running deadline task, CPU1 has one running low priority RT task or idle, CPU3 has one running high priority RT task. When another mid priority RT task is woken on CPU3, it will be pushed to CPU0, while it is reasonable to put it on CPU1. Worse still, this disturbs(causing sched IPI, rq locking, etc) higher-class deadline tasks on CPU0.
This patch eliminates this issue by filtering CPUs that have runnable deadline tasks, using cpudl->free_cpus in cpupri_find() if there's any deadline task.
NOTE: We want to make the most use of percpu local_cpu_mask to save an extra mask allocation, so always passing a non-NULL lowest_mask to cpupri_find().
--- kernel/sched/core.c | 3 ++- kernel/sched/cpupri.c | 29 +++++++++++------------------ kernel/sched/cpupri.h | 3 ++- kernel/sched/rt.c | 14 ++++++++++---- 4 files changed, 25 insertions(+), 24 deletions(-)
kernel/sched/core.c +2 -1
@@ -5644,8 +5644,9 @@ static int init_rootdomain(struct root_domain *rd)if (cpudl_init (&rd->cpudl) !=0 ) goto free_dlo_mask; -if (cpupri_init (&rd->cpupri) !=0 ) +if (cpupri_init (&rd->cpupri, &rd->cpudl) !=0 ) goto free_rto_mask; +return 0 ; free_rto_mask:
kernel/sched/cpupri.c +11 -18
@@ -31,6 +31,7 @@#include <linux/sched.h> #include <linux/sched/rt.h> #include <linux/slab.h> +#include "cpudeadline.h" #include "cpupri.h" /* Convert between a 140 based task->prio, and our 102 based cpupri */
@@ -54,7 +55,7 @@ static int convert_prio(int prio)* cpupri_find - find the best (lowest-pri) CPU in the system * @cp: The cpupri context * @p: The task -* @lowest_mask: A mask to fill in with selected CPUs (or NULL) +* @lowest_mask: A mask to fill in with selected CPUs (not NULL) * * Note: This function returns the recommended CPUs as calculated during the * current invocation. By the time the call returns, the CPUs may have in
@@ -75,6 +76,7 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,for (idx =0 ; idx < task_pri; idx++) {struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; +struct cpudl *dl = cp->cpudl;int skip =0 ;if (!atomic_read (&(vec)->count))
@@ -103,24 +105,12 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,if (skip)continue ; -if (cpumask_any_and (&p->cpus_allowed, vec->mask) >= nr_cpu_ids) +cpumask_and (lowest_mask, &p->cpus_allowed, vec->mask); +if (dl->size) +cpumask_and (lowest_mask, lowest_mask, dl->free_cpus); +if (cpumask_any (lowest_mask) >= nr_cpu_ids)continue ; -if (lowest_mask) { -cpumask_and (lowest_mask, &p->cpus_allowed, vec->mask); - -/* -* We have to ensure that we have at least one bit -* still set in the array, since the map could have -* been concurrently emptied between the first and -* second reads of vec->mask. If we hit this -* condition, simply act as though we never hit this -* priority level and continue on. -*/ -if (cpumask_any (lowest_mask) >= nr_cpu_ids) -continue ; - } -return 1 ; }
@@ -202,10 +192,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)/** * cpupri_init - initialize the cpupri structure * @cp: The cpupri context +* @cpudl: The cpudl context of the same root domain * * Return: -ENOMEM on memory allocation failure. */ -int cpupri_init (struct cpupri *cp) +int cpupri_init (struct cpupri *cp,struct cpudl *cpudl) {int i;
@@ -226,6 +217,8 @@ int cpupri_init(struct cpupri *cp)for_each_possible_cpu (i) cp->cpu_to_pri[i] =CPUPRI_INVALID ; + cp->cpudl = cpudl; +return 0 ; cleanup:
kernel/sched/cpupri.h +2 -1
@@ -18,13 +18,14 @@ struct cpupri_vec {struct cpupri {struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES ];int *cpu_to_pri; +struct cpudl *cpudl; };#ifdef CONFIG_SMP int cpupri_find (struct cpupri *cp,struct task_struct *p,struct cpumask *lowest_mask);void cpupri_set (struct cpupri *cp,int cpu,int pri); -int cpupri_init (struct cpupri *cp); +int cpupri_init (struct cpupri *cp,struct cpudl *cpudl);void cpupri_cleanup (struct cpupri *cp);#endif
kernel/sched/rt.c +10 -4
@@ -1354,14 +1354,22 @@ out:return cpu; } +static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); +static void check_preempt_equal_prio (struct rq *rq,struct task_struct *p) { +struct cpumask *lowest_mask =this_cpu_cpumask_var_ptr (local_cpu_mask); + +/* Make sure the mask is initialized first */ +if (unlikely (!lowest_mask)) +return ; +/* * Current can't be migrated, useless to reschedule, * let's hope p can move out. */ if (rq->curr->nr_cpus_allowed ==1 || - !cpupri_find (&rq->rd->cpupri, rq->curr,NULL )) + !cpupri_find (&rq->rd->cpupri, rq->curr, lowest_mask))return ;/*
@@ -1369,7 +1377,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)* see if it is pushed or pulled somewhere else. */ if (p->nr_cpus_allowed !=1 - &&cpupri_find (&rq->rd->cpupri, p,NULL )) + &&cpupri_find (&rq->rd->cpupri, p, lowest_mask))return ;/*
@@ -1531,8 +1539,6 @@ static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)return NULL ; } -static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); -static int find_lowest_rq (struct task_struct *task) {struct sched_domain *sd;
Signature · 1 line
1.9.1
Ping Juri
Quoted 97 lines
From: Xunlei Pang <[email protected]>
In
check_preempt_equal_dl(),cpudl_find()is called with a NULLlater_mask, thuscpudl_find()here doesn't checkcpudl::free_cpusat all.This patch takles this issue by always passing a non-NULL
later_masktocpudl_find(), thereby fixing this issue.Signed-off-by Xunlei Pang <[email protected]>--- kernel/sched/cpudeadline.c | 8 +++----- kernel/sched/deadline.c | 15 +++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-)kernel/sched/cpudeadline.c +3
-5@@ -97,7 +97,7 @@ static inline int cpudl_maximum(struct cpudl *cp)* cpudl_find - find the best (later-dl) CPU in the system * @cp: the cpudl max-heap context * @p: the task -* @later_mask: a mask to fill in with the selected CPUs (or NULL) +* @later_mask: a mask to fill in with the selected CPUs (not NULL) * * Returns: int - best CPU (heap maximum if suitable) */ @@ -107,15 +107,13 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,int best_cpu =-1 ;const struct sched_dl_entity *dl_se = &p->dl; -if (later_mask && -cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { +if (cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { best_cpu =cpumask_any (later_mask); goto out; }else if (cpumask_test_cpu (cpudl_maximum (cp), &p->cpus_allowed) &&dl_time_before (dl_se->deadline, cp->elements[0 ].dl)) { best_cpu =cpudl_maximum (cp); -if (later_mask) -cpumask_set_cpu (best_cpu, later_mask); +cpumask_set_cpu (best_cpu, later_mask); } out:kernel/sched/deadline.c +11
-5@@ -964,14 +964,23 @@ out:return cpu; } +static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); +static void check_preempt_equal_dl (struct rq *rq,struct task_struct *p) { +struct cpumask *later_mask = +this_cpu_cpumask_var_ptr (local_cpu_mask_dl); + +/* Make sure the mask is initialized first */ +if (unlikely (!later_mask)) +return ; +/* * Current can't be migrated, useless to reschedule, * let's hope p can move out. */ if (rq->curr->nr_cpus_allowed ==1 || -cpudl_find (&rq->rd->cpudl, rq->curr,NULL ) ==-1 ) +cpudl_find (&rq->rd->cpudl, rq->curr, later_mask) ==-1 )return ;/* @@ -979,7 +988,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)* see if it is pushed or pulled somewhere else. */ if (p->nr_cpus_allowed !=1 && -cpudl_find (&rq->rd->cpudl, p,NULL ) !=-1 ) +cpudl_find (&rq->rd->cpudl, p, later_mask) !=-1 )return ;resched_curr (rq);@@ -1173,8 +1182,6 @@ next_node:return NULL ; } -static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); -static int find_later_rq (struct task_struct *task) {struct sched_domain *sd; --1.9.1