per-cpu timer changes

@@ -329,6 +329,21 @@ typedef struct
 } PACKED vcpu_info_t;                   /* 8 + arch */
 
 /*
+ * Xen/kernel shared data
+ *  per cpu timing information.
+ */
+typedef struct time_info_st
+{
+    u32			time_version1;
+    u32			time_version2;
+    tsc_timestamp_t	tsc_timestamp;	/* TSC at last update */
+    u64			system_time;	/* time, in nanoseconds, since boot */
+    u64			cpu_freq;	/* CPU frequency (Hz) */
+    u32			wc_sec;		/* Secs  00:00:00 UTC,  Jan 1, 1970. */
+    u32			wc_usec;	/* Usecs 00:00:00 UTC,  Jan 1, 1970. */
+} PACKED time_info_t;
+
+/*
  * Xen/kernel shared data -- pointer provided in start_info.
  * NB. We expect that this struct is smaller than a page.
  */

@@ -403,6 +418,7 @@ typedef struct shared_info_st
 
     arch_shared_info_t arch;
 
+    time_info_t vcpu_time[MAX_VIRT_CPUS];
 } PACKED shared_info_t;
 
 /*

@@ -53,6 +53,7 @@ string_param("sched", opt_sched);
 /* Various timer handlers. */
 static void s_timer_fn(void *unused);
 static void t_timer_fn(void *unused);
+static void tsc_timer_fn(void *unused);
 static void dom_timer_fn(void *data);
 
 /* This is global for now so that private implementations can reach it */

@@ -76,6 +77,7 @@ static struct scheduler ops;
 
 /* Per-CPU periodic timer sends an event to the currently-executing domain. */
 static struct ac_timer t_timer[NR_CPUS]; 
+static struct ac_timer tsc_timer[NR_CPUS]; 
 
 void free_domain_struct(struct domain *d)
 {

@@ -487,6 +489,7 @@ int idle_cpu(int cpu)
  * Timers: the scheduler utilises a number of timers
  * - s_timer: per CPU timer for preemption and scheduling decisions
  * - t_timer: per CPU periodic timer to send timer interrupt to current dom
+ * - tsc_timer: per CPU periodic timer to update time bases
  * - dom_timer: per domain timer to specifiy timeout values
  ****************************************************************************/
 

@@ -516,6 +519,17 @@ static void t_timer_fn(void *unused)
     set_ac_timer(&t_timer[cpu], NOW() + MILLISECS(10));
 }
 
+/* Periodic tick timer: update time bases for per-cpu timing. */
+static void tsc_timer_fn(void *unused)
+{
+    unsigned int        cpu = current->processor;
+
+    extern void percpu_ticks(void);
+    percpu_ticks();
+
+    set_ac_timer(&tsc_timer[cpu], NOW() + MILLISECS(250));
+}
+
 /* Domain timer function, sends a virtual timer interrupt to domain */
 static void dom_timer_fn(void *data)
 {

@@ -537,6 +551,7 @@ void __init scheduler_init(void)
         spin_lock_init(&schedule_data[i].schedule_lock);
         init_ac_timer(&schedule_data[i].s_timer, s_timer_fn, NULL, i);
         init_ac_timer(&t_timer[i], t_timer_fn, NULL, i);
+        init_ac_timer(&tsc_timer[i], tsc_timer_fn, NULL, i);
     }
 
     schedule_data[0].curr = idle_task[0];

@@ -566,6 +581,9 @@ void schedulers_start(void) 
 {   
     t_timer_fn(0);
     smp_call_function((void *)t_timer_fn, NULL, 1, 1);
+
+    tsc_timer_fn(0);
+    smp_call_function((void *)tsc_timer_fn, NULL, 1, 1);
 }
 
 void dump_runq(unsigned char key)

@@ -431,6 +431,7 @@ void __init start_secondary(void *unused
 
 	extern void percpu_traps_init(void);
 	extern void cpu_init(void);
+	extern void setup_percpu_time(void);
 
 	set_current(idle_task[cpu]);
 	set_processor_id(cpu);

@@ -452,6 +453,7 @@ void __init start_secondary(void *unused
 
 	setup_secondary_APIC_clock();
 	enable_APIC_timer();
+	setup_percpu_time();
 
 	/*
 	 * low-memory mappings have been cleared, flush them from

139 lines not on this page · raw message

204 lines not on this page · raw message

@@ -10,6 +10,7 @@
 #include <linux/cpufreq.h>
 #include <linux/string.h>
 #include <linux/jiffies.h>
+#include <linux/percpu.h>
 
 #include <asm/timer.h>
 #include <asm/io.h>

@@ -35,8 +36,8 @@ extern spinlock_t i8253_lock;
 
 static int use_tsc;
 
-static unsigned long long monotonic_base;
-static u32 monotonic_offset;
+static DEFINE_PER_CPU(unsigned long long, monotonic_base);
+static DEFINE_PER_CPU(u32, monotonic_offset);
 static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
 
 /* convert from cycles(64bits) => nanoseconds (64bits)

@@ -74,8 +75,20 @@ static inline unsigned long long cycles_
  */
 static unsigned long fast_gettimeoffset_quotient;
 
-extern u32 shadow_tsc_stamp;
-extern u64 shadow_system_time;
+extern DEFINE_PER_CPU(u64, shadow_tsc_stamp);
+extern DEFINE_PER_CPU(u64, shadow_system_time);
+
+unsigned long long get_full_tsc_offset(void)
+{
+	unsigned long long tsc;
+
+	/* Read the Time Stamp Counter */
+	rdtscll(tsc);
+
+	tsc -= per_cpu(shadow_tsc_stamp, smp_processor_id());
+
+	return cycles_2_ns(tsc);
+}
 
 static unsigned long get_offset_tsc(void)
 {

@@ -86,7 +99,7 @@ static unsigned long get_offset_tsc(void
 	rdtsc(eax,edx);
 
 	/* .. relative to previous jiffy (32 bits is enough) */
-	eax -= shadow_tsc_stamp;
+	eax -= per_cpu(shadow_tsc_stamp, smp_processor_id());
 
 	/*
          * Time offset = (tsc_low delta) * fast_gettimeoffset_quotient

@@ -110,12 +123,13 @@ static unsigned long long monotonic_cloc
 {
 	unsigned long long last_offset, this_offset, base;
 	unsigned seq;
+	int cpu = smp_processor_id();
 	
 	/* atomically read monotonic base & last_offset */
 	do {
 		seq = read_seqbegin(&monotonic_lock);
-		last_offset = monotonic_offset;
-		base = monotonic_base;
+		last_offset = per_cpu(monotonic_offset, cpu);
+		base = per_cpu(monotonic_base, cpu);
 	} while (read_seqretry(&monotonic_lock, seq));
 
 	/* Read the Time Stamp Counter */

@@ -152,11 +166,12 @@ unsigned long long sched_clock(void)
 
 static void mark_offset_tsc(void)
 {
+	int cpu = smp_processor_id();
 
 	/* update the monotonic base value */
 	write_seqlock(&monotonic_lock);
-	monotonic_base = shadow_system_time;
-	monotonic_offset = shadow_tsc_stamp;
+	per_cpu(monotonic_base, cpu) = per_cpu(shadow_system_time, cpu);
+	per_cpu(monotonic_offset, cpu) = per_cpu(shadow_tsc_stamp, cpu);
 	write_sequnlock(&monotonic_lock);
 }