i386 No Idle HZ aka dynticks 051228

Index: linux-2.6.15-rc7-dt/include/linux/dyn-tick.h
===================================================================
@@ -0,0 +1,170 @@
+/*
+ * linux/include/linux/dyn-tick.h
+ *
+ * Copyright (C) 2004 Nokia Corporation
+ * Written by Tony Lindgen <[email protected]> and
+ * Tuukka Tikkanen <[email protected]>
+ * Rewritten by Con Kolivas <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _DYN_TICK_TIMER_H
+#define _DYN_TICK_TIMER_H
+
+#include <linux/interrupt.h>
+
+#define DYN_TICK_SKIPPING	(1 << 2)
+#define DYN_TICK_ENABLED	(1 << 1)
+#define DYN_TICK_SUITABLE	(1 << 0)
+
+/* Don't skip longer than NMI  */
+#define DYN_TICK_MAX_SKIP	(HZ * 5)
+
+struct dyn_tick_timer {
+	spinlock_t lock;
+
+	/* dyn_tick init */
+	int (*arch_init)(void);
+	/* Enables dynamic tick */
+	int (*arch_enable)(void);
+	/* Disables dynamic tick */
+	int (*arch_disable)(void);
+	/* Reprograms the timer */
+	void (*arch_reprogram)(unsigned long);
+	/* Function called when all cpus are idle, passing the idle duration */
+	void (*arch_all_cpus_idle)(unsigned int);
+
+	unsigned short state;		/* Current state */
+	unsigned int min_skip;		/* Min number of ticks to skip */
+	unsigned int max_skip;		/* Max number of ticks to skip */
+	unsigned long tick;		/* The next earliest tick */
+};
+
+typedef struct {
+	unsigned long next_tick;	/* Next tick we're skipping to */
+	unsigned long skip;		/* Ticks we're currently skipping */
+	unsigned int nohz_cpu;		/* This cpu is idle */
+} dyn_tick_data;
+
+extern struct dyn_tick_timer *dyn_tick;
+extern spinlock_t *dyn_tick_lock;
+
+extern void dyn_tick_register(struct dyn_tick_timer *new_timer);
+
+#ifdef CONFIG_NO_IDLE_HZ
+DECLARE_PER_CPU(dyn_tick_data, dyn_cpu);
+extern dyn_tick_data dyn_cpu;
+extern int dyn_tick_enabled(void);
+extern int dyn_tick_skipping(void);
+extern int dyn_tick_allcpus_skipping(void);
+extern int dyn_tick_current_skip(void);
+extern unsigned long dyn_tick_next_tick(void);
+extern void timer_dyn_reprogram(void);
+extern void dyn_early_reprogram(unsigned int delta);
+extern void set_dyn_tick_limits(unsigned int max_skip, unsigned int min_skip);
+
+/*
+ * The apparently redundant per_cpu nohz_cpu value is tested in this
+ * function and this is where we can avoid the cache thrashing of testing
+ * nohz_cpu_mask when possible.
+ */
+static inline int test_nohz_cpu(void)
+{
+	return __get_cpu_var(dyn_cpu).nohz_cpu;
+}
+
+/*
+ * This cpu is busy, clear the nohz_cpu value, test to see if all were idle
+ * till now. Returns whether all cpus were idle or not.
+ */
+static inline int clear_nohz_cpu(int cpu)
+{
+	int ret = 0;
+
+	if (!test_nohz_cpu())
+		return ret;
+	dyn_tick->tick = 0;
+	if (cpus_equal(nohz_cpu_mask, cpu_online_map)) {
+		dyn_tick->state &= ~DYN_TICK_SKIPPING;
+		ret = 1;
+	}
+	__get_cpu_var(dyn_cpu).next_tick = 0;
+	__get_cpu_var(dyn_cpu).nohz_cpu = 0;
+	cpu_clear(cpu, nohz_cpu_mask);
+	return ret;
+}
+
+/*
+ * This cpu has fallen idle, set the nohz_cpu value, test to see if all are
+ * idle, and if so do dyn_tick->arch_all_cpus_idle()
+ */
+static inline void set_nohz_cpu(int cpu)
+{
+	if (dyn_tick->tick <= jiffies ||
+		__get_cpu_var(dyn_cpu).next_tick < dyn_tick->tick)
+			dyn_tick->tick = __get_cpu_var(dyn_cpu).next_tick;
+
+	if (!test_nohz_cpu()) {
+		__get_cpu_var(dyn_cpu).nohz_cpu = 1;
+		cpu_set(cpu, nohz_cpu_mask);
+	}
+	if (cpus_equal(nohz_cpu_mask, cpu_online_map)) {
+		dyn_tick->state |= DYN_TICK_SKIPPING;
+		dyn_tick->arch_all_cpus_idle(dyn_tick->tick - jiffies);
+	}
+}
+
+#ifdef CONFIG_TIMER_INFO
+extern int account_timer(unsigned long function, unsigned long data);
+#else
+static inline int account_timer(unsigned long function, unsigned long data)
+{
+	return 0;
+}
+#endif
+
+#else	/* CONFIG_NO_IDLE_HZ */
+static inline int dyn_tick_enabled(void)
+{
+	return 0;
+}
+
+static inline int dyn_tick_skipping(void)
+{
+	return 0;
+}
+
+static inline int dyn_tick_allcpus_skipping(void)
+{
+	return 0;
+}
+
+static inline int dyn_tick_current_skip(void)
+{
+	return 0;
+}
+
+static inline unsigned long dyn_tick_next_tick(void)
+{
+	return 0;
+}
+
+static inline void set_dyn_tick_limits(unsigned int max_skip,
+	unsigned int min_skip)
+{
+}
+
+static inline void dyn_early_reprogram(unsigned int delta)
+{
+}
+
+static inline int account_timer(unsigned long function, unsigned long data)
+{
+	return 0;
+}
+#endif	/* CONFIG_NO_IDLE_HZ */
+
+#endif	/* _DYN_TICK_TIMER_H */

Index: linux-2.6.15-rc7-dt/include/linux/timer.h
===================================================================
@@ -97,5 +97,6 @@ static inline void add_timer(struct time
 extern void init_timers(void);
 extern void run_local_timers(void);
 extern void it_real_fn(unsigned long);
+extern void conditional_run_local_timers(void);
 
 #endif

304 lines not on this page · raw message

Index: linux-2.6.15-rc7-dt/kernel/timer.c
===================================================================
@@ -33,6 +33,7 @@
 #include <linux/posix-timers.h>
 #include <linux/cpu.h>
 #include <linux/syscalls.h>
+#include <linux/dyn-tick.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>

@@ -540,6 +541,8 @@ found:
 				expires = nte->expires;
 		}
 	}
+	account_timer((unsigned long)nte->function, nte->data);
+
 	spin_unlock(&base->t_base.lock);
 	return expires;
 }

@@ -869,6 +872,13 @@ void run_local_timers(void)
 	raise_softirq(TIMER_SOFTIRQ);
 }
 
+void conditional_run_local_timers(void)
+{
+	tvec_base_t *base  = &__get_cpu_var(tvec_bases);
+
+	if (base->timer_jiffies != jiffies)
+		run_local_timers();
+}
 /*
  * Called by the timer interrupt. xtime_lock must already be taken
  * by the timer IRQ!

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/time.c
===================================================================
@@ -46,6 +46,7 @@
 #include <linux/bcd.h>
 #include <linux/efi.h>
 #include <linux/mca.h>
+#include <linux/dyn-tick.h>
 
 #include <asm/io.h>
 #include <asm/smp.h>

@@ -56,6 +57,7 @@
 #include <asm/uaccess.h>
 #include <asm/processor.h>
 #include <asm/timer.h>
+#include <asm/dyn-tick.h>
 
 #include "mach_time.h"
 

@@ -245,7 +247,7 @@ EXPORT_SYMBOL(profile_pc);
  * timer_interrupt() needs to keep up the real-time clock,
  * as well as call the "do_timer()" routine every clocktick
  */
-static inline void do_timer_interrupt(int irq, struct pt_regs *regs)
+static inline void do_timer_interrupt(int irq, struct pt_regs *regs, int lost)
 {
 #ifdef CONFIG_X86_IO_APIC
 	if (timer_ack) {

@@ -263,7 +265,8 @@ static inline void do_timer_interrupt(in
 	}
 #endif
 
-	do_timer_interrupt_hook(regs);
+	if (!dyn_tick_enabled() || lost)
+		do_timer_interrupt_hook(regs);
 
 
 	if (MCA_bus) {

@@ -288,6 +291,8 @@ static inline void do_timer_interrupt(in
  */
 irqreturn_t timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 {
+	int lost;
+
 	/*
 	 * Here we are in the timer irq handler. We just have irqs locally
 	 * disabled but we don't know if the timer_bh is running on the other

@@ -297,9 +302,9 @@ irqreturn_t timer_interrupt(int irq, voi
 	 */
 	write_seqlock(&xtime_lock);
 
-	cur_timer->mark_offset();
- 
-	do_timer_interrupt(irq, regs);
+	lost = cur_timer->mark_offset();
+
+	do_timer_interrupt(irq, regs, lost);
 
 	write_sequnlock(&xtime_lock);
 	return IRQ_HANDLED;

@@ -425,7 +430,7 @@ static struct sysdev_class timer_sysclas
 
 
 /* XXX this driverfs stuff should probably go elsewhere later -john */
-static struct sys_device device_timer = {
+struct sys_device device_timer = {
 	.id	= 0,
 	.cls	= &timer_sysclass,
 };

@@ -481,5 +486,7 @@ void __init time_init(void)
 	cur_timer = select_timer();
 	printk(KERN_INFO "Using %s for high-res timesource\n",cur_timer->name);
 
+	dyn_tick_time_init(cur_timer);
+
 	time_init_hook();
 }

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/timers/timer_cyclone.c
===================================================================
@@ -45,7 +45,7 @@ static seqlock_t monotonic_lock = SEQLOC
 	} while (high != cyclone_timer[1]);
 
 
-static void mark_offset_cyclone(void)
+static int mark_offset_cyclone(void)
 {
 	unsigned long lost, delay;
 	unsigned long delta = last_cyclone_low;

@@ -101,6 +101,8 @@ static void mark_offset_cyclone(void)
 	 */
 	if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
 		jiffies_64++;
+
+	return lost;
 }
 
 static unsigned long get_offset_cyclone(void)

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/timers/timer_hpet.c
===================================================================
@@ -101,7 +101,7 @@ static unsigned long get_offset_hpet(voi
 	return edx;
 }
 
-static void mark_offset_hpet(void)
+static int mark_offset_hpet(void)
 {
 	unsigned long long this_offset, last_offset;
 	unsigned long offset;

@@ -124,6 +124,8 @@ static void mark_offset_hpet(void)
 	this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
 	monotonic_base += cycles_2_ns(this_offset - last_offset);
 	write_sequnlock(&monotonic_lock);
+
+	return 1;
 }
 
 static void delay_hpet(unsigned long loops)

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/timers/timer_none.c
===================================================================
@@ -1,9 +1,10 @@
 #include <linux/init.h>
 #include <asm/timer.h>
 
-static void mark_offset_none(void)
+static int mark_offset_none(void)
 {
 	/* nothing needed */
+	return 1;
 }
 
 static unsigned long get_offset_none(void)

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/timers/timer_pit.c
===================================================================
@@ -32,9 +32,10 @@ static int __init init_pit(char* overrid
 	return 0;
 }
 
-static void mark_offset_pit(void)
+static int mark_offset_pit(void)
 {
 	/* nothing needed */
+	return 1;
 }
 
 static unsigned long long monotonic_clock_pit(void)

@@ -148,6 +149,25 @@ static unsigned long get_offset_pit(void
 	return count;
 }
 
+/*
+ * Reprograms the next timer interrupt
+ * PIT timer reprogramming code taken from APM code.
+ * Note that PIT timer is a 16-bit timer.
+ * Called with irqs already disabled.
+ */
+void reprogram_pit_timer(unsigned long jiffies_to_skip)
+{
+	int skip = jiffies_to_skip * LATCH;
+
+	if (skip > 0xffff)
+		skip = 0xffff;
+
+	spin_lock(&i8253_lock);
+	outb_p(0x34, PIT_MODE);		/* binary, mode 2, LSB/MSB, ch 0 */
+	outb_p(skip & 0xff, PIT_CH0);	/* LSB */
+	outb(skip >> 8, PIT_CH0);	/* MSB */
+	spin_unlock(&i8253_lock);
+}
 
 /* tsc timer_opts struct */
 struct timer_opts timer_pit = {

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/timers/timer_pm.c
===================================================================
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/init.h>
+#include <linux/dyn-tick.h>
 #include <asm/types.h>
 #include <asm/timer.h>
 #include <asm/smp.h>

@@ -28,7 +29,7 @@
 #define PMTMR_TICKS_PER_SEC 3579545
 #define PMTMR_EXPECTED_RATE \
   ((CALIBRATE_LATCH * (PMTMR_TICKS_PER_SEC >> 10)) / (CLOCK_TICK_RATE>>10))
-
+#define PMTMR_TICKS_PER_JIFFY (PMTMR_EXPECTED_RATE / (CALIBRATE_LATCH/LATCH))
 
 /* The I/O port the PMTMR resides at.
  * The location is detected during setup_arch(),

@@ -127,7 +128,13 @@ pm_good:
 	if (verify_pmtmr_rate() != 0)
 		return -ENODEV;
 
+	printk("Using %u PM timer ticks per jiffy \n", PMTMR_TICKS_PER_JIFFY);
+
+	offset_tick = read_pmtmr();
+	setup_pit_timer();
+
 	init_cpu_khz();
+	set_dyn_tick_limits((0xFFFFFF / (286 * 1000000)) * 1024 * HZ, 0);
 	return 0;
 }
 

@@ -148,10 +155,9 @@ static inline u32 cyc2us(u32 cycles)
  * this gets called during each timer interrupt
  *   - Called while holding the writer xtime_lock
  */
-static void mark_offset_pmtmr(void)
+static int mark_offset_pmtmr(void)
 {
 	u32 lost, delta, last_offset;
-	static int first_run = 1;
 	last_offset = offset_tick;
 
 	write_seqlock(&monotonic_lock);

@@ -161,29 +167,20 @@ static void mark_offset_pmtmr(void)
 	/* calculate tick interval */
 	delta = (offset_tick - last_offset) & ACPI_PM_MASK;
 
-	/* convert to usecs */
-	delta = cyc2us(delta);
-
 	/* update the monotonic base value */
-	monotonic_base += delta * NSEC_PER_USEC;
+	monotonic_base += cyc2us(delta) * NSEC_PER_USEC;
 	write_sequnlock(&monotonic_lock);
 
 	/* convert to ticks */
 	delta += offset_delay;
-	lost = delta / (USEC_PER_SEC / HZ);
-	offset_delay = delta % (USEC_PER_SEC / HZ);
-
+	lost = delta / PMTMR_TICKS_PER_JIFFY;
+	offset_delay = delta % PMTMR_TICKS_PER_JIFFY;
 
 	/* compensate for lost ticks */
 	if (lost >= 2)
 		jiffies_64 += lost - 1;
 
-	/* don't calculate delay for first run,
-	   or if we've got less then a tick */
-	if (first_run || (lost < 1)) {
-		first_run = 0;
-		offset_delay = 0;
-	}
+	return lost;
 }
 
 static int pmtmr_resume(void)

@@ -192,6 +189,7 @@ static int pmtmr_resume(void)
 	/* Assume this is the last mark offset time */
 	offset_tick = read_pmtmr();
 	write_sequnlock(&monotonic_lock);
+	offset_delay = 0;
 	return 0;
 }
 

@@ -243,7 +241,7 @@ static unsigned long get_offset_pmtmr(vo
 	now = read_pmtmr();
 	delta = (now - offset)&ACPI_PM_MASK;
 
-	return (unsigned long) offset_delay + cyc2us(delta);
+	return (unsigned long) cyc2us(delta + offset_delay);
 }
 
 

143 lines not on this page · raw message

Index: linux-2.6.15-rc7-dt/include/asm-i386/timer.h
===================================================================
@@ -1,5 +1,6 @@
 #ifndef _ASMi386_TIMER_H
 #define _ASMi386_TIMER_H
+#include <linux/jiffies.h>
 #include <linux/init.h>
 #include <linux/pm.h>
 

@@ -19,7 +20,7 @@
  */
 struct timer_opts {
 	char* name;
-	void (*mark_offset)(void);
+	int (*mark_offset)(void);
 	unsigned long (*get_offset)(void);
 	unsigned long long (*monotonic_clock)(void);
 	void (*delay)(unsigned long);

@@ -38,6 +39,7 @@ struct init_timer_opts {
 extern struct timer_opts* __init select_timer(void);
 extern void clock_fallback(void);
 void setup_pit_timer(void);
+extern void reprogram_pit_timer(unsigned long jiffies_to_skip);
 
 /* Modifiers for buggy PIT handling */
 

@@ -67,4 +69,37 @@ extern unsigned long calibrate_tsc_hpet(
 #ifdef CONFIG_X86_PM_TIMER
 extern struct init_timer_opts timer_pmtmr_init;
 #endif
+#ifdef CONFIG_NO_IDLE_HZ
+static inline void tsc_sanity_check(int lost)
+{
+}
+#else /* CONFIG_NO_IDLE_HZ */
+extern void cpufreq_delayed_get(void);
+
+static inline void tsc_sanity_check(int lost)
+{
+	static int lost_count = 0;
+
+	if (lost >= 2) {
+		jiffies_64 += lost-1;
+
+		/* sanity check to ensure we're not always losing ticks */
+		if (lost_count++ > 100) {
+			printk(KERN_WARNING "Losing too many ticks!\n");
+			printk(KERN_WARNING "TSC cannot be used as a timesource.  \n");
+			printk(KERN_WARNING "Possible reasons for this are:\n");
+			printk(KERN_WARNING "  You're running with Speedstep,\n");
+			printk(KERN_WARNING "  You don't have DMA enabled for your hard disk (see hdparm),\n");
+			printk(KERN_WARNING "  Incorrect TSC synchronization on an SMP system (see dmesg).\n");
+			printk(KERN_WARNING "Falling back to a sane timesource now.\n");
+
+			clock_fallback();
+		}
+		/* ... but give the TSC a fair chance */
+		if (lost_count > 25)
+			cpufreq_delayed_get();
+	} else
+		lost_count = 0;
+}
+#endif /* CONFIG_NO_IDLE_HZ */
 #endif

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/apic.c
===================================================================
@@ -26,6 +26,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/sysdev.h>
 #include <linux/cpu.h>
+#include <linux/dyn-tick.h>
 
 #include <asm/atomic.h>
 #include <asm/smp.h>

@@ -35,10 +36,12 @@
 #include <asm/arch_hooks.h>
 #include <asm/hpet.h>
 #include <asm/i8253.h>
+#include <asm/dyn-tick.h>
 
 #include <mach_apic.h>
 
 #include "io_ports.h"
+#include "do_timer.h"
 
 /*
  * Knob to control our willingness to enable the local APIC.

@@ -932,6 +935,8 @@ void (*wait_timer_tick)(void) __devinitd
 
 #define APIC_DIVISOR 16
 
+static u32 apic_timer_val __read_mostly;
+
 static void __setup_APIC_LVTT(unsigned int clocks)
 {
 	unsigned int lvtt_value, tmp_value, ver;

@@ -950,7 +955,9 @@ static void __setup_APIC_LVTT(unsigned i
 				& ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE))
 				| APIC_TDR_DIV_16);
 
-	apic_write_around(APIC_TMICT, clocks/APIC_DIVISOR);
+	apic_timer_val = clocks / APIC_DIVISOR;
+
+	apic_write_around(APIC_TMICT, apic_timer_val);
 }
 
 static void __devinit setup_APIC_timer(unsigned int clocks)

@@ -970,6 +977,17 @@ static void __devinit setup_APIC_timer(u
 }
 
 /*
+ * Used by NO_IDLE_HZ to skip ticks on idle CPUs. Called with IRQs already
+ * disabled
+ */
+void reprogram_apic_timer(unsigned long count)
+{
+	count = count * apic_timer_val;
+	apic_read(APIC_TMICT);
+	apic_write_around(APIC_TMICT, count);
+}
+
+/*
  * In this function we calibrate APIC bus clocks to the external
  * timer. Unfortunately we cannot use jiffies and the timer irq
  * to calibrate, since some later bootup code depends on getting

@@ -1064,6 +1082,10 @@ void __init setup_boot_APIC_clock(void)
 	 */
 	setup_APIC_timer(calibration_result);
 
+	setup_dyn_tick_use_apic(calibration_result);
+	set_dyn_tick_limits((0xFFFFFFFF / calibration_result) * APIC_DIVISOR,
+		2);
+
 	local_irq_restore(flags);
 }
 

@@ -1072,7 +1094,7 @@ void __devinit setup_secondary_APIC_cloc
 	setup_APIC_timer(calibration_result);
 }
 
-void __devinit disable_APIC_timer(void)
+void disable_APIC_timer(void)
 {
 	if (using_apic_timer) {
 		unsigned long v;

@@ -1132,10 +1154,12 @@ int setup_profiling_timer(unsigned int m
  * value into /proc/profile.
  */
 
-inline void smp_local_timer_interrupt(struct pt_regs * regs)
+void smp_local_timer_interrupt(struct pt_regs * regs)
 {
 	int cpu = smp_processor_id();
 
+	dyn_tick_interrupt(regs);
+
 	profile_tick(CPU_PROFILING, regs);
 	if (--per_cpu(prof_counter, cpu) <= 0) {
 		/*

@@ -1202,6 +1226,7 @@ fastcall void smp_apic_timer_interrupt(s
 	 * interrupt lock, which is the WrongThing (tm) to do.
 	 */
 	irq_enter();
+
 	smp_local_timer_interrupt(regs);
 	irq_exit();
 }

@@ -1214,6 +1239,9 @@ fastcall void smp_spurious_interrupt(str
 	unsigned long v;
 
 	irq_enter();
+
+	dyn_tick_interrupt(regs);
+
 	/*
 	 * Check if this really is a spurious interrupt and ACK it
 	 * if it is a vectored one.  Just in case...

@@ -1238,6 +1266,9 @@ fastcall void smp_error_interrupt(struct
 	unsigned long v, v1;
 
 	irq_enter();
+
+	dyn_tick_interrupt(regs);
+
 	/* First tickle the hardware, only then report what went on. -- REW */
 	v = apic_read(APIC_ESR);
 	apic_write(APIC_ESR, 0);

263 lines not on this page · raw message

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/irq.c
===================================================================
@@ -18,6 +18,8 @@
 #include <linux/notifier.h>
 #include <linux/cpu.h>
 #include <linux/delay.h>
+#include <linux/dyn-tick.h>
+#include <asm/dyn-tick.h>
 
 DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_maxaligned_in_smp;
 EXPORT_PER_CPU_SYMBOL(irq_stat);

@@ -76,6 +78,8 @@ fastcall unsigned int do_IRQ(struct pt_r
 	}
 #endif
 
+	dyn_tick_interrupt(regs);
+
 #ifdef CONFIG_4KSTACKS
 
 	curctx = (union irq_ctx *) current_thread_info();

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/process.c
===================================================================
@@ -39,6 +39,7 @@
 #include <linux/ptrace.h>
 #include <linux/random.h>
 #include <linux/kprobes.h>
+#include <linux/dyn-tick.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>

@@ -56,6 +57,7 @@
 
 #include <asm/tlbflush.h>
 #include <asm/cpu.h>
+#include <asm/dyn-tick.h>
 
 asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 

@@ -194,6 +196,8 @@ void cpu_idle(void)
 				play_dead();
 
 			__get_cpu_var(irq_stat).idle_timestamp = jiffies;
+			idle_reprogram_timer();
+
 			idle();
 		}
 		preempt_enable_no_resched();

Index: linux-2.6.15-rc7-dt/arch/i386/kernel/smp.c
===================================================================
@@ -20,9 +20,11 @@
 #include <linux/interrupt.h>
 #include <linux/cpu.h>
 #include <linux/module.h>
+#include <linux/dyn-tick.h>
 
 #include <asm/mtrr.h>
 #include <asm/tlbflush.h>
+#include <asm/dyn-tick.h>
 #include <mach_apic.h>
 
 /*

@@ -313,6 +315,8 @@ fastcall void smp_invalidate_interrupt(s
 {
 	unsigned long cpu;
 
+	dyn_tick_interrupt(regs);
+
 	cpu = get_cpu();
 
 	if (!cpu_isset(cpu, flush_cpumask))

@@ -600,6 +604,8 @@ void smp_send_stop(void)
 fastcall void smp_reschedule_interrupt(struct pt_regs *regs)
 {
 	ack_APIC_irq();
+
+	dyn_tick_interrupt(regs);
 }
 
 fastcall void smp_call_function_interrupt(struct pt_regs *regs)

@@ -609,6 +615,9 @@ fastcall void smp_call_function_interrup
 	int wait = call_data->wait;
 
 	ack_APIC_irq();
+
+	dyn_tick_interrupt(regs);
+
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function

Index: linux-2.6.15-rc7-dt/include/asm-i386/apic.h
===================================================================
@@ -121,6 +121,7 @@ extern void nmi_watchdog_tick (struct pt
 extern int APIC_init_uniprocessor (void);
 extern void disable_APIC_timer(void);
 extern void enable_APIC_timer(void);
+extern void reprogram_apic_timer(unsigned long count);
 
 extern void enable_NMI_through_LVT0 (void * dummy);
 

@@ -134,6 +135,7 @@ extern int disable_timer_pin_1;
 
 #else /* !CONFIG_X86_LOCAL_APIC */
 static inline void lapic_shutdown(void) { }
+static inline void reprogram_apic_timer(unsigned long count) { }
 
 #endif /* !CONFIG_X86_LOCAL_APIC */
 

69 lines not on this page · raw message

234 lines not on this page · raw message

131 lines not on this page · raw message

Index: linux-2.6.15-rc7-dt/include/acpi/processor.h
===================================================================
@@ -51,6 +51,7 @@ struct acpi_processor_cx {
 	u32 latency_ticks;
 	u32 power;
 	u32 usage;
+	u64 time;
 	struct acpi_processor_cx_policy promotion;
 	struct acpi_processor_cx_policy demotion;
 };

@@ -60,6 +61,8 @@ struct acpi_processor_power {
 	unsigned long bm_check_timestamp;
 	u32 default_state;
 	u32 bm_activity;
+	u16 last_sleep;
+	u16 quick_promotion;
 	int count;
 	struct acpi_processor_cx states[ACPI_PROCESSOR_MAX_POWER];
 

Index: linux-2.6.15-rc7-dt/drivers/input/serio/i8042.h
===================================================================
@@ -44,7 +44,7 @@
  * polling.
  */
 
-#define I8042_POLL_PERIOD	HZ/20
+#define I8042_POLL_PERIOD	HZ/5
 
 /*
  * Status register bits.

Index: working-tree/drivers/acpi/processor_idle.c
===================================================================
@@ -3,7 +3,7 @@
  *
  *  Copyright (C) 2001, 2002 Andy Grover <[email protected]>
  *  Copyright (C) 2001, 2002 Paul Diefenbaugh <[email protected]>
- *  Copyright (C) 2004       Dominik Brodowski <[email protected]>
+ *  Copyright (C) 2004, 2005 Dominik Brodowski <[email protected]>
  *  Copyright (C) 2004  Anil S Keshavamurthy <[email protected]>
  *  			- Added processor hotplug support
  *  Copyright (C) 2005  Venkatesh Pallipadi <[email protected]>

@@ -224,21 +224,15 @@ static void acpi_processor_idle(void)
 		u32 bm_status = 0;
 		unsigned long diff = jiffies - pr->power.bm_check_timestamp;
 
-		if (diff > 32)
-			diff = 32;
+		if (diff > 31)
+			diff = 31;
 
-		while (diff) {
-			/* if we didn't get called, assume there was busmaster activity */
-			diff--;
-			if (diff)
-				pr->power.bm_activity |= 0x1;
-			pr->power.bm_activity <<= 1;
-		}
+		pr->power.bm_activity <<= diff;
 
 		acpi_get_register(ACPI_BITREG_BUS_MASTER_STATUS,
 				  &bm_status, ACPI_MTX_DO_NOT_LOCK);
 		if (bm_status) {
-			pr->power.bm_activity++;
+			pr->power.bm_activity |= 0x1;
 			acpi_set_register(ACPI_BITREG_BUS_MASTER_STATUS,
 					  1, ACPI_MTX_DO_NOT_LOCK);
 		}

@@ -250,7 +244,7 @@ static void acpi_processor_idle(void)
 		else if (errata.piix4.bmisx) {
 			if ((inb_p(errata.piix4.bmisx + 0x02) & 0x01)
 			    || (inb_p(errata.piix4.bmisx + 0x0A) & 0x01))
-				pr->power.bm_activity++;
+				pr->power.bm_activity |= 0x1;
 		}
 
 		pr->power.bm_check_timestamp = jiffies;

Index: working-tree/drivers/acpi/processor_idle.c
===================================================================
@@ -250,10 +250,10 @@ static void acpi_processor_idle(void)
 		pr->power.bm_check_timestamp = jiffies;
 
 		/*
-		 * Apply bus mastering demotion policy.  Automatically demote
+		 * If bus mastering is or was active this jiffy, demote
 		 * to avoid a faulty transition.  Note that the processor
 		 * won't enter a low-power state during this call (to this
-		 * funciton) but should upon the next.
+		 * function) but should upon the next.
 		 *
 		 * TBD: A better policy might be to fallback to the demotion
 		 *      state (use it for this quantum only) istead of

@@ -261,7 +261,8 @@ static void acpi_processor_idle(void)
 		 *      qualification.  This may, however, introduce DMA
 		 *      issues (e.g. floppy DMA transfer overrun/underrun).
 		 */
-		if (pr->power.bm_activity & cx->demotion.threshold.bm) {
+		if ((pr->power.bm_activity & 0x1) &&
+		    cx->demotion.threshold.bm) {
 			local_irq_enable();
 			next_state = cx->demotion.state;
 			goto end;

Index: working-tree/drivers/acpi/processor_idle.c
===================================================================
@@ -280,8 +280,6 @@ static void acpi_processor_idle(void)
 		cx = &pr->power.states[ACPI_STATE_C1];
 #endif
 
-	cx->usage++;
-
 	/*
 	 * Sleep:
 	 * ------

@@ -379,6 +377,9 @@ static void acpi_processor_idle(void)
 		local_irq_enable();
 		return;
 	}
+	cx->usage++;
+	if ((cx->type != ACPI_STATE_C1) && (sleep_ticks > 0))
+		cx->time += sleep_ticks;
 
 	next_state = pr->power.state;
 

@@ -993,9 +994,10 @@ static int acpi_processor_power_seq_show
 		else
 			seq_puts(seq, "demotion[--] ");
 
-		seq_printf(seq, "latency[%03d] usage[%08d]\n",
+		seq_printf(seq, "latency[%03d] usage[%08d] duration[%020llu]\n",
 			   pr->power.states[i].latency,
-			   pr->power.states[i].usage);
+			   pr->power.states[i].usage,
+			   pr->power.states[i].time);
 	}
 
       end:

Index: working-tree/include/acpi/processor.h
===================================================================
@@ -51,6 +51,7 @@ struct acpi_processor_cx {
 	u32 latency_ticks;
 	u32 power;
 	u32 usage;
+	u64 time;
 	struct acpi_processor_cx_policy promotion;
 	struct acpi_processor_cx_policy demotion;
 };

Index: working-tree/drivers/acpi/processor_idle.c
===================================================================
@@ -38,6 +38,7 @@
 #include <linux/dmi.h>
 #include <linux/moduleparam.h>
 #include <linux/sched.h>	/* need_resched() */
+#include <linux/dyn-tick.h>
 
 #include <asm/io.h>
 #include <asm/uaccess.h>

@@ -60,6 +61,8 @@ module_param(max_cstate, uint, 0644);
 static unsigned int nocst = 0;
 module_param(nocst, uint, 0000);
 
+#define BM_JIFFIES	(HZ >= 800 ? 2 : 1)
+
 /*
  * bm_history -- bit-mask with a bit per jiffy of bus-master activity
  * 1000 HZ: 0xFFFFFFFF: 32 jiffies = 32ms

@@ -264,6 +267,8 @@ static void acpi_processor_idle(void)
 		if ((pr->power.bm_activity & 0x1) &&
 		    cx->demotion.threshold.bm) {
 			local_irq_enable();
+			if (!pr->power.pre_bm_state)
+				pr->power.pre_bm_state = cx;
 			next_state = cx->demotion.state;
 			goto end;
 		}

@@ -281,6 +286,69 @@ static void acpi_processor_idle(void)
 #endif
 
 	/*
+	 * Some special policy tweaks for dynamic ticks
+	 */
+	if (dyn_tick_enabled()) {
+	       /*
+		* Kick-back promotion: promote to C-State used before bm
+		* activity was detected if
+		*	- we have a pre-bm-state
+		*	- we do not have bus mastering at the moment
+  	  	*  	- we're scheduled to sleep at least BM_JIFFIES now
+		*/
+		if (pr->power.pre_bm_state &&
+		    !(pr->power.bm_activity & 0x1) &&
+		    (dyn_tick_current_skip() >= BM_JIFFIES)) {
+			local_irq_enable();
+			next_state = pr->power.pre_bm_state;
+			pr->power.pre_bm_state = NULL;
+			goto end;
+		}
+
+		/*
+		 * Fast-path promotion: promote to higher state if
+		 *   	- we can promote
+		 *	- there is no bm_activity this tick
+		 *	- we slept more than BM_JIFFIES ticks last time
+		 *	- we're scheduled to sleep at least BM_JIFFIES ticks
+ 		 */
+		if (cx->promotion.state &&
+		    !(pr->power.bm_activity & 0x1) &&
+		    (pr->power.last_sleep > BM_JIFFIES) &&
+		    (dyn_tick_current_skip() >= BM_JIFFIES)) {
+			local_irq_enable();
+			next_state = cx->promotion.state;
+			/*
+			 * Super-fast-path: promote to next higher state if
+			 * 	- we can promote
+			 *	- we did sleep longer than 2 * BM_JIFFIES
+			 *	  times last time
+			 *	- we're scheduled to sleep at least 2 *
+			 *	  BM_JIFFIES ticks
+			 */
+			if ((next_state->promotion.state) &&
+			    (pr->power.last_sleep > 2 * BM_JIFFIES) &&
+			    (dyn_tick_current_skip() >= 2 * BM_JIFFIES))
+				next_state = next_state->promotion.state;
+			pr->power.pre_bm_state = NULL;
+			goto end;
+		}
+
+		/*
+		 * Re-program if bm activity is present this jiffy -- we hope
+		 * that it ends soon, so that we can go into a deeper sleep
+		 * type
+		 */
+		if (cx->demotion.state &&
+		    (pr->power.bm_activity & 0x1) &&
+		    (pr->power.bm_check_timestamp == jiffies)) {
+			dyn_early_reprogram(BM_JIFFIES);
+		}
+	}
+
+	pr->power.last_sleep = 0;
+
+	/*
 	 * Sleep:
 	 * ------
 	 * Invoke the current Cx state to put the processor to sleep.

@@ -377,9 +445,13 @@ static void acpi_processor_idle(void)
 		local_irq_enable();
 		return;
 	}
+
+	/* Accounting */
 	cx->usage++;
 	if ((cx->type != ACPI_STATE_C1) && (sleep_ticks > 0))
 		cx->time += sleep_ticks;
+	pr->power.last_sleep = sleep_ticks / (PM_TIMER_FREQUENCY / HZ);
+
 
 	next_state = pr->power.state;
 

@@ -413,10 +485,12 @@ static void acpi_processor_idle(void)
 					     promotion.threshold.bm)) {
 						next_state =
 						    cx->promotion.state;
+						pr->power.pre_bm_state = NULL;
 						goto end;
 					}
 				} else {
 					next_state = cx->promotion.state;
+					pr->power.pre_bm_state = NULL;
 					goto end;
 				}
 			}

@@ -434,6 +508,7 @@ static void acpi_processor_idle(void)
 			cx->demotion.count++;
 			cx->promotion.count = 0;
 			if (cx->demotion.count >= cx->demotion.threshold.count) {
+				pr->power.pre_bm_state = NULL;
 				next_state = cx->demotion.state;
 				goto end;
 			}

Index: working-tree/include/acpi/processor.h
===================================================================
@@ -61,8 +61,10 @@ struct acpi_processor_power {
 	unsigned long bm_check_timestamp;
 	u32 default_state;
 	u32 bm_activity;
+	u16 last_sleep;
 	int count;
 	struct acpi_processor_cx states[ACPI_PROCESSOR_MAX_POWER];
+	struct acpi_processor_cx *pre_bm_state;
 
 	/* the _PDC objects passed by the driver, if any */
 	struct acpi_object_list *pdc;