optimise resched, idle task

RFC PATCH In review Nick Piggin 1 participant 2 messages Apr 2, 2005 – Apr 3, 2005 ACPI

Share & tools

Nick Piggin

author

2005-04-02 03:31 UTC #0 lore raw

RFC PATCH 1/1 optimise resched, idle task

Haven't finished (hardly started) the arch code for this yet (and probably broke ppc64), so it is just an RFC at this stage.

The large CC list is because it is a reasonably big change, so I hope one or two of you smart guys can see if it looks sound before I try to tackle the rest of the arch code and start asking for testers.

This actually improves performance noticably (ie. a % or so) on schedule / wakeup happy benchmarks (tbench, on a dual Xeon with HT using mwait idle).

Signature · 1 line

SUSE Labs, Novell Inc.

Make some changes to the NEED_RESCHED and POLLING_NRFLAG to reduce confusion, and make their semantics rigid. Also have preempt explicitly disabled in idle routines. Improves efficiency of resched_task and some cpu_idle routines.

* In resched_task:
- TIF_NEED_RESCHED is only cleared with the task's runqueue lock held, and as we hold it during resched_task, then there is no need for an atomic test and set there. (The only time this may prevent an IPI is when the task's quantum expires in the timer interrupt - this is a very rare race to bother with in comparison with the cost).

- If TIF_NEED_RESCHED is set, then we don't need to do anything. It won't get unset until the task get's schedule()d off.

- If we are running on the same CPU as the task we resched, then set TIF_NEED_RESCHED and no further action is required.

- If we are running on another CPU, and TIF_POLLING_NRFLAG is *not* set after TIF_NEED_RESCHED has been set, then we need to send an IPI.

Using these rules, we are able to remove the test and set operation in resched_task, and make clear the previously vague semantics of POLLING_NRFLAG.

* In idle routines:
- Enter cpu_idle with preempt disabled. When the need_resched() condition becomes true, explicitly call schedule(). This makes things a bit clearer (IMO), but haven't updated all architectures yet.

- Many do a test and clear of TIF_NEED_RESCHED for some reason. According to the resched_task rules, this isn't needed (and actually breaks the assumption that TIF_NEED_RESCHED is only cleared with the runqueue lock held). So remove that. Generally one less locked memory op when switching to the idle thread.

- Many idle routines clear TIF_POLLING_NRFLAG, and only set it in the inner most polling idle loops. The above resched_task semantics allow it to be set until before the last time need_resched() is checked before going into a halt requiring interrupt wakeup.

Many idle routines simply never enter such a halt, and so POLLING_NRFLAG can be always left set, completely eliminating resched IPIs when rescheduling the idle task.

POLLING_NRFLAG width can be increased, to reduce the chance of resched IPIs.

kernel/sched.c +14 -7

Index: linux-2.6/kernel/sched.c
===================================================================
@@ -796,21 +796,28 @@ static void deactivate_task(struct task_
 #ifdef CONFIG_SMP
 static void resched_task(task_t *p)
 {
-	int need_resched, nrpolling;
+	int cpu;
 
 	assert_spin_locked(&task_rq(p)->lock);
 
-	/* minimise the chance of sending an interrupt to poll_idle() */
-	nrpolling = test_tsk_thread_flag(p,TIF_POLLING_NRFLAG);
-	need_resched = test_and_set_tsk_thread_flag(p,TIF_NEED_RESCHED);
-	nrpolling |= test_tsk_thread_flag(p,TIF_POLLING_NRFLAG);
+	if (test_tsk_thread_flag(p, TIF_NEED_RESCHED))
+		return;
+	
+	set_tsk_thread_flag(p, TIF_NEED_RESCHED);
 
-	if (!need_resched && !nrpolling && (task_cpu(p) != smp_processor_id()))
-		smp_send_reschedule(task_cpu(p));
+	cpu = task_cpu(p);
+	if (cpu == smp_processor_id())
+		return;
+
+	/* NEED_RESCHED must be visible before we test POLLING_NRFLAG */
+	smp_mb();
+	if (!test_tsk_thread_flag(p, TIF_POLLING_NRFLAG))
+		smp_send_reschedule(cpu);
 }
 #else
 static inline void resched_task(task_t *p)
 {
+	assert_spin_locked(&task_rq(p)->lock);
 	set_tsk_need_resched(p);
 }
 #endif

arch/i386/kernel/process.c +36 ~~-47~~

119 lines not on this page · raw message

init/main.c +1 -1

Index: linux-2.6/init/main.c
===================================================================
@@ -382,7 +382,7 @@ static void noinline rest_init(void)
 	kernel_thread(init, NULL, CLONE_FS | CLONE_SIGHAND);
 	numa_default_policy();
 	unlock_kernel();
-	preempt_enable_no_resched();
+	/* Don't re-enable preemption */
 	cpu_idle();
 }

arch/i386/kernel/apm.c +13 -1

Index: linux-2.6/arch/i386/kernel/apm.c
===================================================================
@@ -767,8 +767,20 @@ static int set_system_power_state(u_shor
 static int apm_do_idle(void)
 {
 	u32	eax;
+	u8	ret;
+	int	idled = 0;
 
-	if (apm_bios_call_simple(APM_FUNC_IDLE, 0, 0, &eax)) {
+	clear_thread_flag(TIF_POLLING_NRFLAG);
+	smp_mb__after_clear_bit();
+	if (!need_resched()) {
+		idled = 1;
+		ret = apm_bios_call_simple(APM_FUNC_IDLE, 0, 0, &eax);
+	}
+	set_thread_flag(TIF_POLLING_NRFLAG);
+	if (!idled)
+		return 0;
+
+	if (ret) {
 		static unsigned long t;
 
 		/* This always fails on some SMP boards running UP kernels.

drivers/acpi/processor_idle.c +18 ~~-14~~

Index: linux-2.6/drivers/acpi/processor_idle.c
===================================================================
@@ -162,6 +162,14 @@ acpi_processor_power_activate (
 	return;
 }
 
+static void acpi_safe_halt (void)
+{
+	clear_thread_flag(TIF_POLLING_NRFLAG);
+	smp_mb__after_clear_bit();
+	while (!need_resched())
+		safe_halt();
+	set_thread_flag(TIF_POLLING_NRFLAG);
+}
 
 static void acpi_processor_idle (void)
 {

@@ -171,7 +179,7 @@ static void acpi_processor_idle (void)
 	int			sleep_ticks = 0;
 	u32			t1, t2 = 0;
 
-	pr = processors[_smp_processor_id()];
+	pr = processors[smp_processor_id()];
 	if (!pr)
 		return;

@@ -191,8 +199,13 @@ static void acpi_processor_idle (void)
 	}
 
 	cx = pr->power.state;
-	if (!cx)
-		goto easy_out;
+	if (!cx) {
+		if (pm_idle_save)
+			pm_idle_save();
+		else
+			acpi_safe_halt();
+		return;
+	}
 
 	/*
 	 * Check BM Activity

@@ -272,7 +285,8 @@ static void acpi_processor_idle (void)
 		if (pm_idle_save)
 			pm_idle_save();
 		else
-			safe_halt();
+			acpi_safe_halt();
+
 		/*
                  * TBD: Can't get time duration while in C1, as resumes
 		 *      go to an ISR rather than here.  Need to instrument

@@ -384,16 +398,6 @@ end:
 	 */
 	if (next_state != pr->power.state)
 		acpi_processor_power_activate(pr, next_state);
-
-	return;
-
- easy_out:
-	/* do C1 instead of busy loop */
-	if (pm_idle_save)
-		pm_idle_save();
-	else
-		safe_halt();
-	return;
 }

arch/i386/kernel/smpboot.c +2 -0

Index: linux-2.6/arch/i386/kernel/smpboot.c
===================================================================
@@ -414,6 +414,8 @@ static int cpucount;
  */
 static void __init start_secondary(void *unused)
 {
+	preempt_disable();
+
 	/*
 	 * Dont put anything before smp_callin(), SMP
 	 * booting is too fragile that we want to limit the

arch/x86_64/kernel/process.c +37 ~~-44~~

114 lines not on this page · raw message

arch/ppc64/kernel/idle.c +88 ~~-93~~

261 lines not on this page · raw message

arch/ia64/kernel/process.c +26 ~~-20~~

72 lines not on this page · raw message

arch/ia64/kernel/smpboot.c +2 -0

Index: linux-2.6/arch/ia64/kernel/smpboot.c
===================================================================
@@ -343,6 +343,8 @@ smp_callin (void)
 int __devinit
 start_secondary (void *unused)
 {
+	preempt_disable();
+
 	/* Early console may use I/O ports */
 	ia64_set_kr(IA64_KR_IO_BASE, __pa(ia64_iobase));

arch/ppc64/kernel/smp.c +4 -1

Index: linux-2.6/arch/ppc64/kernel/smp.c
===================================================================
@@ -561,7 +561,10 @@ int __devinit __cpu_up(unsigned int cpu)
 /* Activate a secondary processor. */
 int __devinit start_secondary(void *unused)
 {
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu;
+	
+	preempt_disable();
+	cpu = smp_processor_id();
 
 	atomic_inc(&init_mm.mm_count);
 	current->active_mm = &init_mm;

Nick Piggin

author

2005-04-03 13:27 UTC #1 lore raw

Quoted 3 lines · Nick Piggin

This actually improves performance noticably (ie. a % or so) on schedule / wakeup happy benchmarks (tbench, on a dual Xeon with HT using mwait idle).

Here are some numbers on a 2 socket Xeon with HT and mwait idle.

Average of 3 runs, tbench, single client and single server processes on:
samethread samecpu othercpu

before patch:   188.684 MB/s    189.237 MB/s    172.306 MB/s
after patch :   188.425 MB/s    191.628 MB/s    174.224 MB/s

The improvement on other CPUs should be due to the removal of RMW operations in resched_task.