Skip to contentSkip to conversation

lib/plist: Provide plist_add_head() for nodes with the same prio

PATCH RESEND Superseded Xunlei Pang 4 participants 7 messages Mar 9, 2015 – Apr 1, 2015

Subsystems (2)

v4 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

PATCH RESEND v4 1/3 lib/plist: Provide plist_add_head() for nodes with the same prio

From: Xunlei Pang <[email protected]>

If there're multiple nodes with the same prio as @node, currently plist_add() will add @node behind all of them. Now we need to add @node before all of these nodes for SMP RT scheduler.

This patch adds a common __plist_add() for adding @node before or after existing nodes with the same prio, then adds plist_add_head() and plist_add_tail() inline wrapper functions for convenient uses.

Finally, define plist_add() as plist_add_tail() which has the same behaviour as before.

cc Andrew Morton <[email protected]>
cc Dan Streetman <[email protected]>
---
 include/linux/plist.h | 30 +++++++++++++++++++++++++++++-
 lib/plist.c           | 24 +++++++++++++++++++++---
 2 files changed, 50 insertions(+), 4 deletions(-)
include/linux/plist.h +29 -1
index 9788360..10c834c 100644
@@ -138,7 +138,35 @@ static inline void plist_node_init(struct plist_node *node, int prio)
 	INIT_LIST_HEAD(&node->node_list);
 }
 
-extern void plist_add(struct plist_node *node, struct plist_head *head);
+extern void __plist_add(struct plist_node *node,
+				struct plist_head *head, bool is_head);
+
+/**
+ * plist_add_head - add @node to @head, before all existing same-prio nodes
+ *
+ * @node:	&struct plist_node pointer
+ * @head:	&struct plist_head pointer
+ */
+static inline
+void plist_add_head(struct plist_node *node, struct plist_head *head)
+{
+	__plist_add(node, head, true);
+}
+
+/**
+ * plist_add_tail - add @node to @head, after all existing same-prio nodes
+ *
+ * @node:	&struct plist_node pointer
+ * @head:	&struct plist_head pointer
+ */
+static inline
+void plist_add_tail(struct plist_node *node, struct plist_head *head)
+{
+	__plist_add(node, head, false);
+}
+
+#define plist_add plist_add_tail
+
 extern void plist_del(struct plist_node *node, struct plist_head *head);
 
 extern void plist_requeue(struct plist_node *node, struct plist_head *head);
lib/plist.c +21 -3
index 3a30c53..d3ebac6 100644
@@ -66,12 +66,16 @@ static void plist_check_head(struct plist_head *head)
 #endif
 
 /**
- * plist_add - add @node to @head
+ * __plist_add - add @node to @head
  *
  * @node:	&struct plist_node pointer
  * @head:	&struct plist_head pointer
+ * @is_head:	bool
+ *
+ * If there're any nodes with the same prio, add @node
+ * behind or before all of them according to @is_head.
  */
-void plist_add(struct plist_node *node, struct plist_head *head)
+void __plist_add(struct plist_node *node, struct plist_head *head, bool is_head)
 {
 	struct plist_node *first, *iter, *prev = NULL;
 	struct list_head *node_next = &head->node_list;
@@ -96,8 +100,22 @@ void plist_add(struct plist_node *node, struct plist_head *head)
 				struct plist_node, prio_list);
 	} while (iter != first);
 
-	if (!prev || prev->prio != node->prio)
+	if (!prev || prev->prio != node->prio) {
 		list_add_tail(&node->prio_list, &iter->prio_list);
+	} else if (is_head) {
+		/*
+		 * prev has the same priority as the node that is
+		 * being added. It is also the first node for this
+		 * priority, but the new node needs to be added ahead of
+		 * it. To accomplish this, insert node right after prev
+		 * in the prio_list and then remove prev from that list.
+		 * Then set node_next to prev->node_list so that the
+		 * new node gets added before prev and not iter.
+		 */
+		list_add(&node->prio_list, &prev->prio_list);
+		list_del_init(&prev->prio_list);
+		node_next = &prev->node_list;
+	}
 ins_node:
 	list_add_tail(&node->node_list, node_next);
 
Signature · 1 line
1.9.1
#1 lore raw

PATCH RESEND v4 2/3 sched/rt: Fix wrong SMP scheduler behavior for equal prio cases

From: Xunlei Pang <[email protected]>

Currently, SMP RT scheduler has some trouble in dealing with equal prio cases.

For example, in check_preempt_equal_prio():
When RT1(current task) gets preempted by RT2, if there is a migratable RT3 with same prio, RT3 will be pushed away instead of RT1 afterwards, because RT1 will be enqueued to the tail of the pushable list when going through succeeding put_prev_task_rt() triggered by resched. This broke FIFO.

Furthermore, this is also problematic for normal preempted cases if there're some rt tasks queued with the same prio as current. Because current will be put behind these tasks in the pushable queue.

So, if a task is running and gets preempted by a higher priority task (or even with same priority for migrating), this patch ensures that it is put ahead of any existing task with the same priority in the pushable queue.

---
 kernel/sched/rt.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
kernel/sched/rt.c +16 -7
index f4d4b07..86cd79f 100644
@@ -347,11 +347,15 @@ static inline void set_post_schedule(struct rq *rq)
 	rq->post_schedule = has_pushable_tasks(rq);
 }
 
-static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static void enqueue_pushable_task(struct rq *rq,
+				struct task_struct *p, bool head)
 {
 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
 	plist_node_init(&p->pushable_tasks, p->prio);
-	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
+	if (head)
+		plist_add_head(&p->pushable_tasks, &rq->rt.pushable_tasks);
+	else
+		plist_add_tail(&p->pushable_tasks, &rq->rt.pushable_tasks);
 
 	/* Update the highest prio pushable task */
 	if (p->prio < rq->rt.highest_prio.next)
@@ -373,7 +377,8 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
 
 #else
 
-static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static inline void enqueue_pushable_task(struct rq *rq,
+					struct task_struct *p, bool head)
 {
 }
 
@@ -1248,7 +1253,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
 	enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
 
 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
-		enqueue_pushable_task(rq, p);
+		enqueue_pushable_task(rq, p, false);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
@@ -1494,8 +1499,12 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 	 * The previous task needs to be made eligible for pushing
 	 * if it is still active
 	 */
-	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
-		enqueue_pushable_task(rq, p);
+	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) {
+		if (task_running(rq, p) && (preempt_count() & PREEMPT_ACTIVE))
+			enqueue_pushable_task(rq, p, true);
+		else
+			enqueue_pushable_task(rq, p, false);
+	}
 }
 
 #ifdef CONFIG_SMP
@@ -1914,7 +1923,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 		rq->rt.rt_nr_migratory--;
 	} else {
 		if (!task_current(rq, p))
-			enqueue_pushable_task(rq, p);
+			enqueue_pushable_task(rq, p, false);
 		rq->rt.rt_nr_migratory++;
 	}
 
Signature · 1 line
1.9.1
#2 lore raw

Re: [PATCH RESEND v4 2/3] sched/rt: Fix wrong SMP scheduler behavior for equal prio cases

Ping Steve

Quoted 96 lines

From: Xunlei Pang <[email protected]>

Currently, SMP RT scheduler has some trouble in dealing with equal prio cases.

For example, in check_preempt_equal_prio():
When RT1(current task) gets preempted by RT2, if there is a migratable RT3 with same prio, RT3 will be pushed away instead of RT1 afterwards, because RT1 will be enqueued to the tail of the pushable list when going through succeeding put_prev_task_rt() triggered by resched. This broke FIFO.

Furthermore, this is also problematic for normal preempted cases if there're some rt tasks queued with the same prio as current. Because current will be put behind these tasks in the pushable queue.

So, if a task is running and gets preempted by a higher priority task (or even with same priority for migrating), this patch ensures that it is put ahead of any existing task with the same priority in the pushable queue.

---
 kernel/sched/rt.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
kernel/sched/rt.c +16 -8
index f4d4b07..86cd79f 100644
@@ -347,11 +347,15 @@ static inline void set_post_schedule(struct rq *rq)
        rq->post_schedule = has_pushable_tasks(rq);
 }
 
-static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static void enqueue_pushable_task(struct rq *rq,
+                               struct task_struct *p, bool head)
 {
        plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
        plist_node_init(&p->pushable_tasks, p->prio);
-       plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
+       if (head)
+               plist_add_head(&p->pushable_tasks, &rq->rt.pushable_tasks);
+       else
+               plist_add_tail(&p->pushable_tasks, &rq->rt.pushable_tasks);
 
        /* Update the highest prio pushable task */
        if (p->prio < rq->rt.highest_prio.next)
@@ -373,7 +377,8 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
 
 #else
 
-static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static inline void enqueue_pushable_task(struct rq *rq,
+                                       struct task_struct *p, bool head)
 {
 }
 
@@ -1248,7 +1253,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
        enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
 
        if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
-               enqueue_pushable_task(rq, p);
+               enqueue_pushable_task(rq, p, false);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
@@ -1494,8 +1499,12 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
         * The previous task needs to be made eligible for pushing
         * if it is still active
         */
-       if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
-               enqueue_pushable_task(rq, p);
+       if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) {
+               if (task_running(rq, p) && (preempt_count() & PREEMPT_ACTIVE))
+                       enqueue_pushable_task(rq, p, true);
+               else
+                       enqueue_pushable_task(rq, p, false);
+       }
 }
 
 #ifdef CONFIG_SMP
@@ -1914,7 +1923,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
                rq->rt.rt_nr_migratory--;
        } else {
                if (!task_current(rq, p))
-                       enqueue_pushable_task(rq, p);
+                       enqueue_pushable_task(rq, p, false);
                rq->rt.rt_nr_migratory++;
        }
 
--

1.9.1

#3 lore raw

Re: [PATCH RESEND v4 2/3] sched/rt: Fix wrong SMP scheduler behavior for equal prio cases

In reply to Xunlei Pang
Quoted 32 lines · On Mon, 9 Mar 2015 15:32:27 +0800 Xunlei Pang <[email protected]> wrote:

From: Xunlei Pang <[email protected]>

Currently, SMP RT scheduler has some trouble in dealing with equal prio cases.

For example, in check_preempt_equal_prio():
When RT1(current task) gets preempted by RT2, if there is a migratable RT3 with same prio, RT3 will be pushed away instead of RT1 afterwards, because RT1 will be enqueued to the tail of the pushable list when going through succeeding put_prev_task_rt() triggered by resched. This broke FIFO.

Furthermore, this is also problematic for normal preempted cases if there're some rt tasks queued with the same prio as current. Because current will be put behind these tasks in the pushable queue.

So, if a task is running and gets preempted by a higher priority task (or even with same priority for migrating), this patch ensures that it is put ahead of any existing task with the same priority in the pushable queue.

---
 kernel/sched/rt.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
kernel/sched/rt.c +0 -0
index f4d4b07..86cd79f 100644

kernel/sched/rt.c static inline void set_post_schedule(struct rq *rq) · line 351

Quoted 6 lines
kernel/sched/rt.c +2 -1
@@ -347,11 +347,15 @@ static inline void set_post_schedule(struct rq *rq)
 	rq->post_schedule = has_pushable_tasks(rq);
 }
 
-static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static void enqueue_pushable_task(struct rq *rq,
+				struct task_struct *p, bool head)

Nit.

static void
enqueue_pushable_task(struct rq *rq, struct task_struct *p, bool head)

Is a better breaking of the line.

Quoted 12 lines
kernel/sched/rt.c +4 -1
 {
 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
 	plist_node_init(&p->pushable_tasks, p->prio);
-	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
+	if (head)
+		plist_add_head(&p->pushable_tasks, &rq->rt.pushable_tasks);
+	else
+		plist_add_tail(&p->pushable_tasks, &rq->rt.pushable_tasks);
 
 	/* Update the highest prio pushable task */
 	if (p->prio < rq->rt.highest_prio.next)

kernel/sched/rt.c static void dequeue_pushable_task(struct rq *rq, struct task_struct *p) · line 381

Quoted 6 lines
kernel/sched/rt.c +2 -1
@@ -373,7 +377,8 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
 
 #else
 
-static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static inline void enqueue_pushable_task(struct rq *rq,
+					struct task_struct *p, bool head)

Same here.

Quoted 13 lines
kernel/sched/rt.c +1 -1
 {
 }
 
@@ -1248,7 +1253,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
 	enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
 
 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
-		enqueue_pushable_task(rq, p);
+		enqueue_pushable_task(rq, p, false);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)

kernel/sched/rt.c static void put_prev_task_rt(struct rq *rq, struct task_struct *p) · line 1503

Quoted 7 lines
kernel/sched/rt.c +2 -2
@@ -1494,8 +1499,12 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 	 * The previous task needs to be made eligible for pushing
 	 * if it is still active
 	 */
-	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
-		enqueue_pushable_task(rq, p);
+	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) {
+		if (task_running(rq, p) && (preempt_count() & PREEMPT_ACTIVE))

put_prev_task_rt() is called by put_prev_task() which is called by several functions: rt_mutex_setprio(), __sched_setscheduler(), sched_setnuma(), migrate_tasks(), and sched_move_task(). It's not part of being preempted.

Now it is also called by pick_next_task_rt() which I'm assuming is what you want it to affect.

The above definitely needs a comment about what it is doing. Also, I'm not so sure we care about testing task_running(). I'm thinking the check for PREEMPT_ACTIVE is good enough, as that would only be set from being called within preempt_schedule().

Also, we could get rid of the if statement and do:

	enqueue_pushable_task(rq, p, !!(preempt_count() & PREEMPT_ACTIVE));

-- Steve

Quoted 16 lines
kernel/sched/rt.c +5 -1
+			enqueue_pushable_task(rq, p, true);
+		else
+			enqueue_pushable_task(rq, p, false);
+	}
 }
 
 #ifdef CONFIG_SMP
@@ -1914,7 +1923,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 		rq->rt.rt_nr_migratory--;
 	} else {
 		if (!task_current(rq, p))
-			enqueue_pushable_task(rq, p);
+			enqueue_pushable_task(rq, p, false);
 		rq->rt.rt_nr_migratory++;
 	}
 
#4 lore raw

Re: [PATCH RESEND v4 2/3] sched/rt: Fix wrong SMP scheduler behavior for equal prio cases

Quoted 44 lines · Steven Rostedt
Quoted 32 lines · On Mon, 9 Mar 2015 15:32:27 +0800 Xunlei Pang <[email protected]> wrote:

From: Xunlei Pang <[email protected]>

Currently, SMP RT scheduler has some trouble in dealing with equal prio cases.

For example, in check_preempt_equal_prio():
When RT1(current task) gets preempted by RT2, if there is a migratable RT3 with same prio, RT3 will be pushed away instead of RT1 afterwards, because RT1 will be enqueued to the tail of the pushable list when going through succeeding put_prev_task_rt() triggered by resched. This broke FIFO.

Furthermore, this is also problematic for normal preempted cases if there're some rt tasks queued with the same prio as current. Because current will be put behind these tasks in the pushable queue.

So, if a task is running and gets preempted by a higher priority task (or even with same priority for migrating), this patch ensures that it is put ahead of any existing task with the same priority in the pushable queue.

---
 kernel/sched/rt.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
kernel/sched/rt.c +0 -0
index f4d4b07..86cd79f 100644

kernel/sched/rt.c static inline void set_post_schedule(struct rq *rq) · line 351

Quoted 6 lines
kernel/sched/rt.c +2 -1
@@ -347,11 +347,15 @@ static inline void set_post_schedule(struct rq *rq)
      rq->post_schedule = has_pushable_tasks(rq);
 }
 
-static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static void enqueue_pushable_task(struct rq *rq,
+                             struct task_struct *p, bool head)

Nit.

Quoted 4 lines

static void
enqueue_pushable_task(struct rq *rq, struct task_struct *p, bool head)

Is a better breaking of the line.

I'll adjust it to be one line.

Quoted 48 lines
Quoted 13 lines
kernel/sched/rt.c +4 -1
 {
      plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
      plist_node_init(&p->pushable_tasks, p->prio);
-     plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
+     if (head)
+             plist_add_head(&p->pushable_tasks, &rq->rt.pushable_tasks);
+     else
+             plist_add_tail(&p->pushable_tasks, &rq->rt.pushable_tasks);
 
      /* Update the highest prio pushable task */
      if (p->prio < rq->rt.highest_prio.next)

kernel/sched/rt.c static void dequeue_pushable_task(struct rq *rq, struct task_struct *p) · line 381

Quoted 6 lines
kernel/sched/rt.c +2 -1
@@ -373,7 +377,8 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
 
 #else
 
-static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
+static inline void enqueue_pushable_task(struct rq *rq,
+                                     struct task_struct *p, bool head)

Same here.

Quoted 13 lines
kernel/sched/rt.c +1 -1
 {
 }
 
@@ -1248,7 +1253,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
      enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
 
      if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
-             enqueue_pushable_task(rq, p);
+             enqueue_pushable_task(rq, p, false);
 }
 
 static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)

kernel/sched/rt.c static void put_prev_task_rt(struct rq *rq, struct task_struct *p) · line 1503

Quoted 7 lines
kernel/sched/rt.c +2 -2
@@ -1494,8 +1499,12 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
       * The previous task needs to be made eligible for pushing
       * if it is still active
       */
-     if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
-             enqueue_pushable_task(rq, p);
+     if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) {
+             if (task_running(rq, p) && (preempt_count() & PREEMPT_ACTIVE))

put_prev_task_rt() is called by put_prev_task() which is called by several functions: rt_mutex_setprio(), __sched_setscheduler(), sched_setnuma(), migrate_tasks(), and sched_move_task(). It's not part of being preempted.

Quoted 7 lines

Now it is also called by pick_next_task_rt() which I'm assuming is what you want it to affect.

The above definitely needs a comment about what it is doing. Also, I'm not so sure we care about testing task_running(). I'm thinking the check for PREEMPT_ACTIVE is good enough, as that would only be set from being called within preempt_schedule().

Indeed.

Quoted 4 lines

Also, we could get rid of the if statement and do:

        enqueue_pushable_task(rq, p, !!(preempt_count() & PREEMPT_ACTIVE));

Agree, will do. Thanks.

Quoted 21 lines

-- Steve

Quoted 16 lines
kernel/sched/rt.c +5 -1
+                     enqueue_pushable_task(rq, p, true);
+             else
+                     enqueue_pushable_task(rq, p, false);
+     }
 }
 
 #ifdef CONFIG_SMP
@@ -1914,7 +1923,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
              rq->rt.rt_nr_migratory--;
      } else {
              if (!task_current(rq, p))
-                     enqueue_pushable_task(rq, p);
+                     enqueue_pushable_task(rq, p, false);
              rq->rt.rt_nr_migratory++;
      }
 
#5 lore raw

PATCH RESEND v4 3/3 sched/rt: Check to push the task when changing its affinity

In reply to Xunlei Pang

From: Xunlei Pang <[email protected]>

We may suffer from extra rt overload rq due to the affinity, so when the affinity of any runnable rt task is changed, we should check to trigger balancing, otherwise it will cause some unnecessary delayed real-time response. Unfortunately, current RT global scheduler doesn't trigger anything.

For example: a 2-cpu system with two runnable FIFO tasks(same rt_priority) bound on CPU0, let's name them rt1(running) and rt2(runnable) respectively; CPU1 has no RTs. Then, someone sets the affinity of rt2 to 0x3(i.e. CPU0 and CPU1), but after this, rt2 still can't be scheduled until rt1 enters schedule(), this definitely causes some/big response latency for rt2.

So, when doing set_cpus_allowed_rt(), if detecting such cases, check to trigger a push behaviour.

---
 kernel/sched/rt.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 68 insertions(+), 10 deletions(-)
kernel/sched/rt.c +68 -10
index 86cd79f..ac048d7 100644
@@ -1433,10 +1433,9 @@ static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
 	return next;
 }
 
-static struct task_struct *_pick_next_task_rt(struct rq *rq)
+static struct task_struct *peek_next_task_rt(struct rq *rq)
 {
 	struct sched_rt_entity *rt_se;
-	struct task_struct *p;
 	struct rt_rq *rt_rq  = &rq->rt;
 
 	do {
@@ -1445,7 +1444,14 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
 		rt_rq = group_rt_rq(rt_se);
 	} while (rt_rq);
 
-	p = rt_task_of(rt_se);
+	return rt_task_of(rt_se);
+}
+
+static inline struct task_struct *_pick_next_task_rt(struct rq *rq)
+{
+	struct task_struct *p;
+
+	p = peek_next_task_rt(rq);
 	p->se.exec_start = rq_clock_task(rq);
 
 	return p;
@@ -1895,28 +1901,74 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 				const struct cpumask *new_mask)
 {
 	struct rq *rq;
-	int weight;
+	int old_weight, new_weight;
+	int preempt_push = 0, direct_push = 0;
 
 	BUG_ON(!rt_task(p));
 
 	if (!task_on_rq_queued(p))
 		return;
 
-	weight = cpumask_weight(new_mask);
+	old_weight = p->nr_cpus_allowed;
+	new_weight = cpumask_weight(new_mask);
+
+	rq = task_rq(p);
+
+	if (new_weight > 1 &&
+	    rt_task(rq->curr) &&
+	    !test_tsk_need_resched(rq->curr)) {
+		/*
+		 * We own p->pi_lock and rq->lock. rq->lock might
+		 * get released when doing direct pushing, however
+		 * p->pi_lock is always held, so it's safe to assign
+		 * new_mask and new_weight to p below.
+		 */
+		if (!task_running(rq, p)) {
+			cpumask_copy(&p->cpus_allowed, new_mask);
+			p->nr_cpus_allowed = new_weight;
+			direct_push = 1;
+		} else if (cpumask_test_cpu(task_cpu(p), new_mask)) {
+			cpumask_copy(&p->cpus_allowed, new_mask);
+			p->nr_cpus_allowed = new_weight;
+			if (!cpupri_find(&rq->rd->cpupri, p, NULL))
+				goto update;
+
+			/*
+			 * At this point, current task gets migratable most
+			 * likely due to the change of its affinity, let's
+			 * figure out if we can migrate it.
+			 *
+			 * Is there any task with the same priority as that
+			 * of current task? If found one, we should resched.
+			 * NOTE: The target may be unpushable.
+			 */
+			if (p->prio == rq->rt.highest_prio.next) {
+				/* One target just in pushable_tasks list. */
+				requeue_task_rt(rq, p, 0);
+				preempt_push = 1;
+			} else if (rq->rt.rt_nr_total > 1) {
+				struct task_struct *next;
+
+				requeue_task_rt(rq, p, 0);
+				next = peek_next_task_rt(rq);
+				if (next != p && next->prio == p->prio)
+					preempt_push = 1;
+			}
+		}
+	}
 
+update:
 	/*
 	 * Only update if the process changes its state from whether it
 	 * can migrate or not.
 	 */
-	if ((p->nr_cpus_allowed > 1) == (weight > 1))
-		return;
-
-	rq = task_rq(p);
+	if ((old_weight > 1) == (new_weight > 1))
+		goto out;
 
 	/*
 	 * The process used to be able to migrate OR it can now migrate
 	 */
-	if (weight <= 1) {
+	if (new_weight <= 1) {
 		if (!task_current(rq, p))
 			dequeue_pushable_task(rq, p);
 		BUG_ON(!rq->rt.rt_nr_migratory);
@@ -1928,6 +1980,12 @@ static void set_cpus_allowed_rt(struct task_struct *p,
 	}
 
 	update_rt_migration(&rq->rt);
+
+out:
+	if (direct_push)
+		push_rt_tasks(rq);
+	else if (preempt_push)
+		resched_curr(rq);
 }
 
 /* Assumes rq->lock is held */
Signature · 1 line
1.9.1
Reviewed-by on v4 1/3
#6 lore raw
In reply to Xunlei Pang
Quoted 94 lines · Xunlei Pang

From: Xunlei Pang <[email protected]>

If there're multiple nodes with the same prio as @node, currently plist_add() will add @node behind all of them. Now we need to add @node before all of these nodes for SMP RT scheduler.

This patch adds a common __plist_add() for adding @node before or after existing nodes with the same prio, then adds plist_add_head() and plist_add_tail() inline wrapper functions for convenient uses.

Finally, define plist_add() as plist_add_tail() which has the same behaviour as before.

cc Andrew Morton <[email protected]>
cc Dan Streetman <[email protected]>
---
 include/linux/plist.h | 30 +++++++++++++++++++++++++++++-
 lib/plist.c           | 24 +++++++++++++++++++++---
 2 files changed, 50 insertions(+), 4 deletions(-)
include/linux/plist.h +29 -1
index 9788360..10c834c 100644
@@ -138,7 +138,35 @@ static inline void plist_node_init(struct plist_node *node, int prio)
        INIT_LIST_HEAD(&node->node_list);
 }
 
-extern void plist_add(struct plist_node *node, struct plist_head *head);
+extern void __plist_add(struct plist_node *node,
+                               struct plist_head *head, bool is_head);
+
+/**
+ * plist_add_head - add @node to @head, before all existing same-prio nodes
+ *
+ * @node:      &struct plist_node pointer
+ * @head:      &struct plist_head pointer
+ */
+static inline
+void plist_add_head(struct plist_node *node, struct plist_head *head)
+{
+       __plist_add(node, head, true);
+}
+
+/**
+ * plist_add_tail - add @node to @head, after all existing same-prio nodes
+ *
+ * @node:      &struct plist_node pointer
+ * @head:      &struct plist_head pointer
+ */
+static inline
+void plist_add_tail(struct plist_node *node, struct plist_head *head)
+{
+       __plist_add(node, head, false);
+}
+
+#define plist_add plist_add_tail
+
 extern void plist_del(struct plist_node *node, struct plist_head *head);
 
 extern void plist_requeue(struct plist_node *node, struct plist_head *head);
lib/plist.c +9 -3
index 3a30c53..d3ebac6 100644
@@ -66,12 +66,16 @@ static void plist_check_head(struct plist_head *head)
 #endif
 
 /**
- * plist_add - add @node to @head
+ * __plist_add - add @node to @head
  *
  * @node:      &struct plist_node pointer
  * @head:      &struct plist_head pointer
+ * @is_head:   bool
+ *
+ * If there're any nodes with the same prio, add @node
+ * behind or before all of them according to @is_head.
  */
-void plist_add(struct plist_node *node, struct plist_head *head)
+void __plist_add(struct plist_node *node, struct plist_head *head, bool is_head)
 {
        struct plist_node *first, *iter, *prev = NULL;
        struct list_head *node_next = &head->node_list;
@@ -96,8 +100,22 @@ void plist_add(struct plist_node *node, struct plist_head *head)
                                struct plist_node, prio_list);
        } while (iter != first);
 
-       if (!prev || prev->prio != node->prio)
+       if (!prev || prev->prio != node->prio) {
                list_add_tail(&node->prio_list, &iter->prio_list);
+       } else if (is_head) {
+               /*

lib/plist.c void plist_add(struct plist_node *node, struct plist_head *head) · line 116

Quoted 10 lines
lib/plist.c +10 -0
+                * prev has the same priority as the node that is
+                * being added. It is also the first node for this
+                * priority, but the new node needs to be added ahead of
+                * it. To accomplish this, insert node right after prev
+                * in the prio_list and then remove prev from that list.
+                * Then set node_next to prev->node_list so that the
+                * new node gets added before prev and not iter.
+                */
+               list_add(&node->prio_list, &prev->prio_list);
+               list_del_init(&prev->prio_list);

I think this can just be

list_replace_init(&prev->prio_list, &node->prio_list);

besides that,

Quoted 9 lines
lib/plist.c +2 -1
+               node_next = &prev->node_list;
+       }
 ins_node:
        list_add_tail(&node->node_list, node_next);
 
--

1.9.1