Skip to contentSkip to conversation

scheduler patches

PATCH In review Peter Zijlstra 2 participants 6 messages Dec 16, 2008 Scheduler

lore
Share & tools
lore
author
#1 lore raw

PATCH 1/3 From: Mike Galbraith <[email protected]>

From: Mike Galbraith <[email protected]>

It was possible to do the preemption check against an old time stamp.

Signed-off-by Mike Galbraith <[email protected]>
Signed-off-by Peter Zijlstra <[email protected]>
---
 kernel/sched.c      |    2 +-
 kernel/sched_fair.c |    7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)
kernel/sched.c +1 -1
index ce55b6a..efe5c6d 100644
@@ -2259,6 +2259,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 
 	smp_wmb();
 	rq = task_rq_lock(p, &flags);
+	update_rq_clock(rq);
 	old_state = p->state;
 	if (!(old_state & state))
 		goto out;

@@ -2316,7 +2317,6 @@ out_activate:
 		schedstat_inc(p, se.nr_wakeups_local);
 	else
 		schedstat_inc(p, se.nr_wakeups_remote);
-	update_rq_clock(rq);
 	activate_task(rq, p, 1);
 	success = 1;
kernel/sched_fair.c +3 -4
index 08ffffd..6ae5115 100644
@@ -1343,12 +1343,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
 {
 	struct task_struct *curr = rq->curr;
 	struct sched_entity *se = &curr->se, *pse = &p->se;
+	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
 
-	if (unlikely(rt_prio(p->prio))) {
-		struct cfs_rq *cfs_rq = task_cfs_rq(curr);
+	update_curr(cfs_rq);
 
-		update_rq_clock(rq);
-		update_curr(cfs_rq);
+	if (unlikely(rt_prio(p->prio))) {
 		resched_task(curr);
 		return;
 	}
author
#2 lore raw

Re: [PATCH 1/3] From: Mike Galbraith <[email protected]>

I suck!

Subject: sched: fix wakeup preemption clock

Quoted 53 lines · Peter Zijlstra

plain text document attachment (sched-fix-wakeup-clock.patch)
From: Mike Galbraith <[email protected]>

It was possible to do the preemption check against an old time stamp.

Signed-off-by Mike Galbraith <[email protected]>
Signed-off-by Peter Zijlstra <[email protected]>
---
 kernel/sched.c      |    2 +-
 kernel/sched_fair.c |    7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)
kernel/sched.c +1 -1
index ce55b6a..efe5c6d 100644
@@ -2259,6 +2259,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 
 	smp_wmb();
 	rq = task_rq_lock(p, &flags);
+	update_rq_clock(rq);
 	old_state = p->state;
 	if (!(old_state & state))
 		goto out;

@@ -2316,7 +2317,6 @@ out_activate:
 		schedstat_inc(p, se.nr_wakeups_local);
 	else
 		schedstat_inc(p, se.nr_wakeups_remote);
-	update_rq_clock(rq);
 	activate_task(rq, p, 1);
 	success = 1;
kernel/sched_fair.c +3 -4
index 08ffffd..6ae5115 100644
@@ -1343,12 +1343,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
 {
 	struct task_struct *curr = rq->curr;
 	struct sched_entity *se = &curr->se, *pse = &p->se;
+	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
 
-	if (unlikely(rt_prio(p->prio))) {
-		struct cfs_rq *cfs_rq = task_cfs_rq(curr);
+	update_curr(cfs_rq);
 
-		update_rq_clock(rq);
-		update_curr(cfs_rq);
+	if (unlikely(rt_prio(p->prio))) {
 		resched_task(curr);
 		return;
 	}
author
#3 lore raw

PATCH 2/3 sched: optimize update_curr()

In reply to Peter Zijlstra

Skip the hard work when there is none.

Signed-off-by Peter Zijlstra <[email protected]>
Acked-by Mike Galbraith <[email protected]>
---
 kernel/sched_fair.c |    2 ++
 1 file changed, 2 insertions(+)
kernel/sched_fair.c +2 -0
Index: linux-2.6/kernel/sched_fair.c
===================================================================
@@ -492,6 +492,8 @@ static void update_curr(struct cfs_rq *c
 	 * overflow on 32 bits):
 	 */
 	delta_exec = (unsigned long)(now - curr->exec_start);
+	if (!delta_exec)
+		return;
 
 	__update_curr(cfs_rq, curr, delta_exec);
 	curr->exec_start = now;
author
#4 lore raw

PATCH 3/3 sched: prefer wakers

In reply to Peter Zijlstra

Prefer tasks that wake other tasks to preempt quickly. This improves performance because more work is available sooner.

The workload that prompted this patch was a kernel build over NFS4 (for some curious and not understood reason we had to revert commit:
18de9735300756e3ca9c361ef58409d8561dfe0d to make any progress at all)

Without this patch a make -j8 bzImage (of x86-64 defconfig) would take 3m30-ish, with this patch we're down to 2m50-ish.

psql-sysbench/mysql-sysbench show a slight improvement in peak performance as well, tbench and vmark seemed to not care.

It is possible to improve upon the build time (to 2m20-ish) but that seriously destroys other benchmarks (just shows that there's more room for tinkering).

Much thanks to Mike who put in a lot of effort to benchmark things and proved a worthy opponent with a competing patch.

Signed-off-by Peter Zijlstra <[email protected]>
Signed-off-by Mike Galbraith <[email protected]>
---
 include/linux/sched.h   |    2 +
 kernel/sched.c          |   30 +++++++++++++++++++++---
 kernel/sched_debug.c    |    1 
 kernel/sched_fair.c     |   59 +++++++++++++++++++++++++++++++++++++++++++-----
 kernel/sched_features.h |    3 +-
 5 files changed, 84 insertions(+), 11 deletions(-)
kernel/sched.c +26 -4
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -1690,17 +1690,35 @@ static void update_avg(u64 *avg, u64 sam
 
 static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
 {
+	if (wakeup)
+		p->se.start_runtime = p->se.sum_exec_runtime;
+
 	sched_info_queued(p);
 	p->sched_class->enqueue_task(rq, p, wakeup);
 	p->se.on_rq = 1;
 }
 
+static void update_avg_wakeup(struct sched_entity *se)
+{
+	u64 sample = se->sum_exec_runtime;
+	if (se->last_wakeup)
+		sample -= se->last_wakeup;
+	else
+		sample -= se->start_runtime;
+	update_avg(&se->avg_wakeup, sample);
+}
+
 static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
 {
-	if (sleep && p->se.last_wakeup) {
-		update_avg(&p->se.avg_overlap,
-			   p->se.sum_exec_runtime - p->se.last_wakeup);
-		p->se.last_wakeup = 0;
+	if (sleep) {
+		if (p->se.last_wakeup) {
+			update_avg(&p->se.avg_overlap,
+				p->se.sum_exec_runtime - p->se.last_wakeup);
+			p->se.last_wakeup = 0;
+		} else {
+			update_avg(&p->se.avg_wakeup,
+				sysctl_sched_wakeup_granularity);
+		}
 	}
 
 	sched_info_dequeued(p);

@@ -2327,6 +2345,8 @@ out_activate:
 		schedstat_inc(p, se.nr_wakeups_remote);
 	activate_task(rq, p, 1);
 	success = 1;
+	if (!in_interrupt())
+		update_avg_wakeup(&current->se);
 
 out_running:
 	trace_sched_wakeup(rq, p);

@@ -2369,6 +2389,8 @@ static void __sched_fork(struct task_str
 	p->se.prev_sum_exec_runtime	= 0;
 	p->se.last_wakeup		= 0;
 	p->se.avg_overlap		= 0;
+	p->se.start_runtime		= 0;
+	p->se.avg_wakeup		= sysctl_sched_wakeup_granularity;
 
 #ifdef CONFIG_SCHEDSTATS
 	p->se.wait_start		= 0;
kernel/sched_fair.c +53 -6

76 lines not on this page · raw message

include/linux/sched.h +2 -0
Index: linux-2.6/include/linux/sched.h
===================================================================
@@ -1024,6 +1024,8 @@ struct sched_entity {
 
 	u64			last_wakeup;
 	u64			avg_overlap;
+	u64			start_runtime;
+	u64			avg_wakeup;
 
 #ifdef CONFIG_SCHEDSTATS
 	u64			wait_start;
kernel/sched_features.h +2 -1
Index: linux-2.6/kernel/sched_features.h
===================================================================
@@ -1,5 +1,5 @@
 SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
-SCHED_FEAT(NORMALIZED_SLEEPER, 1)
+SCHED_FEAT(NORMALIZED_SLEEPER, 0)
 SCHED_FEAT(WAKEUP_PREEMPT, 1)
 SCHED_FEAT(START_DEBIT, 1)
 SCHED_FEAT(AFFINE_WAKEUPS, 1)

@@ -13,3 +13,4 @@ SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
 SCHED_FEAT(ASYM_EFF_LOAD, 1)
 SCHED_FEAT(WAKEUP_OVERLAP, 0)
 SCHED_FEAT(LAST_BUDDY, 1)
+SCHED_FEAT(ADAPTIVE_GRAN, 1)
kernel/sched_debug.c +1 -0
Index: linux-2.6/kernel/sched_debug.c
===================================================================
@@ -384,6 +384,7 @@ void proc_sched_show_task(struct task_st
 	PN(se.vruntime);
 	PN(se.sum_exec_runtime);
 	PN(se.avg_overlap);
+	PN(se.avg_wakeup);
 
 	nr_switches = p->nvcsw + p->nivcsw;
#5 lore raw

Re: [PATCH 3/3] sched: prefer wakers

Quoted 10 lines · Peter Zijlstra

plain text document attachment (wakeup-preempt-fiddle.patch) Prefer tasks that wake other tasks to preempt quickly. This improves performance because more work is available sooner.

The workload that prompted this patch was a kernel build over NFS4 (for some curious and not understood reason we had to revert commit:
18de9735300756e3ca9c361ef58409d8561dfe0d to make any progress at all)

Without this patch a make -j8 bzImage (of x86-64 defconfig) would take 3m30-ish, with this patch we're down to 2m50-ish.

Here, with wimpy 100Mbps nic and my normal config, the improvement is considerably larger. Full make -j8 build went from ~6m17s to ~4m28s.

-Mike