Skip to contentSkip to conversation

sched: Do not stop cpu in set_cpus_allowed_ptr() if task is not running

PATCH In review Kirill Tkhai 6 participants 10 messages Sep 12, 2014 – Sep 19, 2014 Scheduler

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 sched: Do not stop cpu in set_cpus_allowed_ptr() if task is not running

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +32 -15
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+	struct rq *rq = task_rq(p);
+
+	lockdep_assert_held(&rq->lock);
+
+	dequeue_task(rq, p, 0);
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	set_task_cpu(p, new_cpu);
+	raw_spin_unlock(&rq->lock);
+
+	rq = cpu_rq(new_cpu);
+
+	raw_spin_lock(&rq->lock);
+	BUG_ON(task_cpu(p) != new_cpu);
+	p->on_rq = TASK_ON_RQ_QUEUED;
+	enqueue_task(rq, p, 0);
+	check_preempt_curr(rq, p, 0);
+
+	return rq;
+}
+
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
 	if (p->sched_class && p->sched_class->set_cpus_allowed)
@@ -4650,14 +4677,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 		goto out;
 
 	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-	if (task_on_rq_queued(p) || p->state == TASK_WAKING) {
+	if (task_running(rq, p) || p->state == TASK_WAKING) {
 		struct migration_arg arg = { p, dest_cpu };
 		/* Need help from migration thread: drop lock and wait. */
 		task_rq_unlock(rq, p, &flags);
 		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
 		tlb_migrate_finish(p->mm);
 		return 0;
-	}
+	} else if (task_on_rq_queued(p))
+		rq = move_queued_task(p, dest_cpu);
 out:
 	task_rq_unlock(rq, p, &flags);
 
@@ -4700,19 +4728,8 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
 	 * If we're not on a rq, the next wake-up will ensure we're
 	 * placed properly.
 	 */
-	if (task_on_rq_queued(p)) {
-		dequeue_task(rq, p, 0);
-		p->on_rq = TASK_ON_RQ_MIGRATING;
-		set_task_cpu(p, dest_cpu);
-		raw_spin_unlock(&rq->lock);
-
-		rq = cpu_rq(dest_cpu);
-		raw_spin_lock(&rq->lock);
-		BUG_ON(task_rq(p) != rq);
-		p->on_rq = TASK_ON_RQ_QUEUED;
-		enqueue_task(rq, p, 0);
-		check_preempt_curr(rq, p, 0);
-	}
+	if (task_on_rq_queued(p))
+		rq = move_queued_task(p, dest_cpu);
 done:
 	ret = 1;
 fail:
maintainer
#1 lore raw
Quoted 33 lines · Kirill Tkhai

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +16 -0
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+	struct rq *rq = task_rq(p);
+
+	lockdep_assert_held(&rq->lock);
+
+	dequeue_task(rq, p, 0);
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	set_task_cpu(p, new_cpu);
+	raw_spin_unlock(&rq->lock);
+

kernel/sched/core.c void init_idle(struct task_struct *idle, int cpu) · line 4622

Quoted 10 lines
kernel/sched/core.c +10 -0
+	rq = cpu_rq(new_cpu);
+
+	raw_spin_lock(&rq->lock);
+	BUG_ON(task_cpu(p) != new_cpu);
+	p->on_rq = TASK_ON_RQ_QUEUED;
+	enqueue_task(rq, p, 0);
+	check_preempt_curr(rq, p, 0);
+
+	return rq;
+}

We have almost that exact code sequence in __migrate_task() could we share some?

#2 lore raw
Quoted 39 lines · Peter Zijlstra
Quoted 38 lines · Kirill Tkhai

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +21 -0
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+	struct rq *rq = task_rq(p);
+
+	lockdep_assert_held(&rq->lock);
+
+	dequeue_task(rq, p, 0);
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	set_task_cpu(p, new_cpu);
+	raw_spin_unlock(&rq->lock);
+
+	rq = cpu_rq(new_cpu);
+
+	raw_spin_lock(&rq->lock);
+	BUG_ON(task_cpu(p) != new_cpu);
+	p->on_rq = TASK_ON_RQ_QUEUED;
Quoted 9 lines

kernel/sched/core.c void init_idle(struct task_struct *idle, int cpu) · line 4622

Quoted 5 lines
kernel/sched/core.c +5 -0
+	enqueue_task(rq, p, 0);
+	check_preempt_curr(rq, p, 0);
+
+	return rq;
+}

We have almost that exact code sequence in __migrate_task() could we share some?

Peter, haven't I moved the sequence from __migrate_task??

Rechecked again; yes, patch makes __migrate_task using new move_queued_task.

Kirill

maintainer
#3 lore raw
Quoted 4 lines · Kirill Tkhai

On 14.09.2014 13:13, Peter Zijlstra wrote:
Peter, haven't I moved the sequence from __migrate_task??

Rechecked again; yes, patch makes __migrate_task using new move_queued_task.

Yes you have ... urgh.. that's what you get for traveling 9 timezones and trying to do 'work' at 3 am which is really noon.

Lemme try and get my head on straight and have another look.

#4 lore raw
Quoted 11 lines · Peter Zijlstra
Quoted 4 lines · Kirill Tkhai

On 14.09.2014 13:13, Peter Zijlstra wrote:
Peter, haven't I moved the sequence from __migrate_task??

Rechecked again; yes, patch makes __migrate_task using new move_queued_task.

Yes you have ... urgh.. that's what you get for traveling 9 timezones and trying to do 'work' at 3 am which is really noon.

Lemme try and get my head on straight and have another look.

Oh, no problem :) Happy Programmers Day!!!

#5 lore raw
In reply to Kirill Tkhai

Hi Kirill,

Which tree is this patch based on? __migrate_task() does a double_rq_lock/unlock() today in mainline, doesn't it? I don't however see that in your patch.

Regards
Preeti U Murthy

Quoted 95 lines · Kirill Tkhai

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +32 -16
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+       struct rq *rq = task_rq(p);
+
+       lockdep_assert_held(&rq->lock);
+
+       dequeue_task(rq, p, 0);
+       p->on_rq = TASK_ON_RQ_MIGRATING;
+       set_task_cpu(p, new_cpu);
+       raw_spin_unlock(&rq->lock);
+
+       rq = cpu_rq(new_cpu);
+
+       raw_spin_lock(&rq->lock);
+       BUG_ON(task_cpu(p) != new_cpu);
+       p->on_rq = TASK_ON_RQ_QUEUED;
+       enqueue_task(rq, p, 0);
+       check_preempt_curr(rq, p, 0);
+
+       return rq;
+}
+
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
        if (p->sched_class && p->sched_class->set_cpus_allowed)
@@ -4650,14 +4677,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
                goto out;
 
        dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-       if (task_on_rq_queued(p) || p->state == TASK_WAKING) {
+       if (task_running(rq, p) || p->state == TASK_WAKING) {
                struct migration_arg arg = { p, dest_cpu };
                /* Need help from migration thread: drop lock and wait. */
                task_rq_unlock(rq, p, &flags);
                stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
                tlb_migrate_finish(p->mm);
                return 0;
-       }
+       } else if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 out:
        task_rq_unlock(rq, p, &flags);
 
@@ -4700,19 +4728,8 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         * If we're not on a rq, the next wake-up will ensure we're
         * placed properly.
         */
-       if (task_on_rq_queued(p)) {
-               dequeue_task(rq, p, 0);
-               p->on_rq = TASK_ON_RQ_MIGRATING;
-               set_task_cpu(p, dest_cpu);
-               raw_spin_unlock(&rq->lock);
-
-               rq = cpu_rq(dest_cpu);
-               raw_spin_lock(&rq->lock);
-               BUG_ON(task_rq(p) != rq);
-               p->on_rq = TASK_ON_RQ_QUEUED;
-               enqueue_task(rq, p, 0);
-               check_preempt_curr(rq, p, 0);
-       }
+       if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 done:
        ret = 1;
 fail:
 
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#6 lore raw

Hi, Preeti,

В Пн, 15/09/2014 в 15:18 +0530, Preeti Murthy пишет:

Quoted 5 lines

Hi Kirill,

Which tree is this patch based on? __migrate_task() does a double_rq_lock/unlock() today in mainline, doesn't it? I don't however see that in your patch.

it's based on recent tip tree:
https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git/

Also, once I received a build-robot message about my patch makes warning on allyesconfig, and since that time I learned about Peter's tree :)

https://git.kernel.org/cgit/linux/kernel/git/peterz/queue.git

(This is for info, I don't know if it is right to make patches based on in. But really good if you was away for some time and you're interested in recent news without lkml archive reading. I use it :)

Regards,
Kirill

Quoted 96 lines · Kirill Tkhai
Quoted 95 lines

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +32 -16
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+       struct rq *rq = task_rq(p);
+
+       lockdep_assert_held(&rq->lock);
+
+       dequeue_task(rq, p, 0);
+       p->on_rq = TASK_ON_RQ_MIGRATING;
+       set_task_cpu(p, new_cpu);
+       raw_spin_unlock(&rq->lock);
+
+       rq = cpu_rq(new_cpu);
+
+       raw_spin_lock(&rq->lock);
+       BUG_ON(task_cpu(p) != new_cpu);
+       p->on_rq = TASK_ON_RQ_QUEUED;
+       enqueue_task(rq, p, 0);
+       check_preempt_curr(rq, p, 0);
+
+       return rq;
+}
+
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
        if (p->sched_class && p->sched_class->set_cpus_allowed)
@@ -4650,14 +4677,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
                goto out;
 
        dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-       if (task_on_rq_queued(p) || p->state == TASK_WAKING) {
+       if (task_running(rq, p) || p->state == TASK_WAKING) {
                struct migration_arg arg = { p, dest_cpu };
                /* Need help from migration thread: drop lock and wait. */
                task_rq_unlock(rq, p, &flags);
                stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
                tlb_migrate_finish(p->mm);
                return 0;
-       }
+       } else if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 out:
        task_rq_unlock(rq, p, &flags);
 
@@ -4700,19 +4728,8 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         * If we're not on a rq, the next wake-up will ensure we're
         * placed properly.
         */
-       if (task_on_rq_queued(p)) {
-               dequeue_task(rq, p, 0);
-               p->on_rq = TASK_ON_RQ_MIGRATING;
-               set_task_cpu(p, dest_cpu);
-               raw_spin_unlock(&rq->lock);
-
-               rq = cpu_rq(dest_cpu);
-               raw_spin_lock(&rq->lock);
-               BUG_ON(task_rq(p) != rq);
-               p->on_rq = TASK_ON_RQ_QUEUED;
-               enqueue_task(rq, p, 0);
-               check_preempt_curr(rq, p, 0);
-       }
+       if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 done:
        ret = 1;
 fail:
 
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

maintainer
#7 lore raw
Quoted 16 lines · Kirill Tkhai

Hi, Preeti,

В Пн, 15/09/2014 в 15:18 +0530, Preeti Murthy пишет:

Quoted 5 lines

Hi Kirill,

Which tree is this patch based on? __migrate_task() does a double_rq_lock/unlock() today in mainline, doesn't it? I don't however see that in your patch.

it's based on recent tip tree:
https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git/

Also, once I received a build-robot message about my patch makes warning on allyesconfig, and since that time I learned about Peter's tree :)

Quoted 6 lines

https://git.kernel.org/cgit/linux/kernel/git/peterz/queue.git

(This is for info, I don't know if it is right to make patches based on in. But really good if you was away for some time and you're interested in recent news without lkml archive reading. I use it :)

Its mostly a scratch tree to get some build coverage and to show people what all is queued up in my quilt tree. The tree is fully recreated every time I export my queue and at times I forget to export.

So feel free to look at, but basing work on it might not be the bestest idea. Ideally things should find their way into -tip fairly quickly (or get dropped on the ground if they come apart, but that usually goes paired with an email telling you about the borkage).

#8 lore raw
In reply to Kirill Tkhai
Quoted 16 lines · Kirill Tkhai

Hi, Preeti,

В Пн, 15/09/2014 в 15:18 +0530, Preeti Murthy пишет:

Quoted 5 lines

Hi Kirill,

Which tree is this patch based on? __migrate_task() does a double_rq_lock/unlock() today in mainline, doesn't it? I don't however see that in your patch.

it's based on recent tip tree:
https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git/

Also, once I received a build-robot message about my patch makes warning on allyesconfig, and since that time I learned about Peter's tree :)

Quoted 6 lines

https://git.kernel.org/cgit/linux/kernel/git/peterz/queue.git

(This is for info, I don't know if it is right to make patches based on in. But really good if you was away for some time and you're interested in recent news without lkml archive reading. I use it :)

Ok I checked the patch against tip tree and the patch looks good to me.

Reviewed-by; Preeti U Murthy <[email protected]>

Thanks!

Regards
Preeti U Murthy

Quoted 102 lines

Regards,
Kirill

Quoted 96 lines · Kirill Tkhai
Quoted 95 lines

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c |   47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +32 -16
index d4399b4..dbbba26 100644
@@ -4594,6 +4594,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+       struct rq *rq = task_rq(p);
+
+       lockdep_assert_held(&rq->lock);
+
+       dequeue_task(rq, p, 0);
+       p->on_rq = TASK_ON_RQ_MIGRATING;
+       set_task_cpu(p, new_cpu);
+       raw_spin_unlock(&rq->lock);
+
+       rq = cpu_rq(new_cpu);
+
+       raw_spin_lock(&rq->lock);
+       BUG_ON(task_cpu(p) != new_cpu);
+       p->on_rq = TASK_ON_RQ_QUEUED;
+       enqueue_task(rq, p, 0);
+       check_preempt_curr(rq, p, 0);
+
+       return rq;
+}
+
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
        if (p->sched_class && p->sched_class->set_cpus_allowed)
@@ -4650,14 +4677,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
                goto out;
 
        dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-       if (task_on_rq_queued(p) || p->state == TASK_WAKING) {
+       if (task_running(rq, p) || p->state == TASK_WAKING) {
                struct migration_arg arg = { p, dest_cpu };
                /* Need help from migration thread: drop lock and wait. */
                task_rq_unlock(rq, p, &flags);
                stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
                tlb_migrate_finish(p->mm);
                return 0;
-       }
+       } else if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 out:
        task_rq_unlock(rq, p, &flags);
 
@@ -4700,19 +4728,8 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         * If we're not on a rq, the next wake-up will ensure we're
         * placed properly.
         */
-       if (task_on_rq_queued(p)) {
-               dequeue_task(rq, p, 0);
-               p->on_rq = TASK_ON_RQ_MIGRATING;
-               set_task_cpu(p, dest_cpu);
-               raw_spin_unlock(&rq->lock);
-
-               rq = cpu_rq(dest_cpu);
-               raw_spin_lock(&rq->lock);
-               BUG_ON(task_rq(p) != rq);
-               p->on_rq = TASK_ON_RQ_QUEUED;
-               enqueue_task(rq, p, 0);
-               check_preempt_curr(rq, p, 0);
-       }
+       if (task_on_rq_queued(p))
+               rq = move_queued_task(p, dest_cpu);
 done:
        ret = 1;
 fail:
 
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#9 lore raw
In reply to Kirill Tkhai
sched: Do not stop cpu in set_cpus_allowed_ptr() if task is not running
Commit-ID:  a15b12ac36ad4e7b856a4ae54937ae26a51aebad
Gitweb:     http://git.kernel.org/tip/a15b12ac36ad4e7b856a4ae54937ae26a51aebad
Author:     Kirill Tkhai <[email protected]>
AuthorDate: Fri, 12 Sep 2014 15:03:34 +0400
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 19 Sep 2014 12:35:21 +0200

sched: Do not stop cpu in set_cpus_allowed_ptr() if task is not running

If a task is queued but not running on it rq, we can simply migrate it without migration thread and switching of context.

---
 kernel/sched/core.c | 47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)
kernel/sched/core.c +32 -15
index 5536397..4b1ddeb 100644
@@ -4629,6 +4629,33 @@ void init_idle(struct task_struct *idle, int cpu)
 }
 
 #ifdef CONFIG_SMP
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released.
+ */
+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
+{
+	struct rq *rq = task_rq(p);
+
+	lockdep_assert_held(&rq->lock);
+
+	dequeue_task(rq, p, 0);
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	set_task_cpu(p, new_cpu);
+	raw_spin_unlock(&rq->lock);
+
+	rq = cpu_rq(new_cpu);
+
+	raw_spin_lock(&rq->lock);
+	BUG_ON(task_cpu(p) != new_cpu);
+	p->on_rq = TASK_ON_RQ_QUEUED;
+	enqueue_task(rq, p, 0);
+	check_preempt_curr(rq, p, 0);
+
+	return rq;
+}
+
 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
 {
 	if (p->sched_class && p->sched_class->set_cpus_allowed)
@@ -4685,14 +4712,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 		goto out;
 
 	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-	if (task_on_rq_queued(p) || p->state == TASK_WAKING) {
+	if (task_running(rq, p) || p->state == TASK_WAKING) {
 		struct migration_arg arg = { p, dest_cpu };
 		/* Need help from migration thread: drop lock and wait. */
 		task_rq_unlock(rq, p, &flags);
 		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
 		tlb_migrate_finish(p->mm);
 		return 0;
-	}
+	} else if (task_on_rq_queued(p))
+		rq = move_queued_task(p, dest_cpu);
 out:
 	task_rq_unlock(rq, p, &flags);
 
@@ -4735,19 +4763,8 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
 	 * If we're not on a rq, the next wake-up will ensure we're
 	 * placed properly.
 	 */
-	if (task_on_rq_queued(p)) {
-		dequeue_task(rq, p, 0);
-		p->on_rq = TASK_ON_RQ_MIGRATING;
-		set_task_cpu(p, dest_cpu);
-		raw_spin_unlock(&rq->lock);
-
-		rq = cpu_rq(dest_cpu);
-		raw_spin_lock(&rq->lock);
-		BUG_ON(task_rq(p) != rq);
-		p->on_rq = TASK_ON_RQ_QUEUED;
-		enqueue_task(rq, p, 0);
-		check_preempt_curr(rq, p, 0);
-	}
+	if (task_on_rq_queued(p))
+		rq = move_queued_task(p, dest_cpu);
 done:
 	ret = 1;
 fail: