mcount and latency tracing utility -v8

Index: linux-mcount.git/kernel/printk.c
===================================================================
@@ -590,9 +590,11 @@ static int have_callable_console(void)
  * @fmt: format string
  *
  * This is printk().  It can be called from any context.  We want it to work.
- * Be aware of the fact that if oops_in_progress is not set, we might try to
- * wake klogd up which could deadlock on runqueue lock if printk() is called
- * from scheduler code.
+ *
+ * Note: if printk() is called with the runqueue lock held, it will not wake
+ * up the klogd. This is to avoid a deadlock from calling printk() in schedule
+ * with the runqueue lock held and having the wake_up grab the runqueue lock
+ * as well.
  *
  * We try to grab the console_sem.  If we succeed, it's easy - we log the output and
  * call the console drivers.  If we fail to get the semaphore we place the output

@@ -1001,7 +1003,11 @@ void release_console_sem(void)
 	console_locked = 0;
 	up(&console_sem);
 	spin_unlock_irqrestore(&logbuf_lock, flags);
-	if (wake_klogd)
+	/*
+	 * If we try to wake up klogd while printing with the runqueue lock
+	 * held, this will deadlock.
+	 */
+	if (wake_klogd && !runqueue_is_locked())
 		wake_up_klogd();
 }
 EXPORT_SYMBOL(release_console_sem);

Index: linux-mcount.git/include/linux/sched.h
===================================================================
@@ -222,6 +222,8 @@ extern void sched_init_smp(void);
 extern void init_idle(struct task_struct *idle, int cpu);
 extern void init_idle_bootup_task(struct task_struct *idle);
 
+extern int runqueue_is_locked(void);
+
 extern cpumask_t nohz_cpu_mask;
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ)
 extern int select_nohz_load_balancer(int cpu);

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -621,6 +621,24 @@ unsigned long rt_needs_cpu(int cpu)
 # define const_debug static const
 #endif
 
+/**
+ * runqueue_is_locked
+ *
+ * Returns true if the current cpu runqueue is locked.
+ * This interface allows printk to be called with the runqueue lock
+ * held and know whether or not it is OK to wake up the klogd.
+ */
+int runqueue_is_locked(void)
+{
+	int cpu = get_cpu();
+	struct rq *rq = cpu_rq(cpu);
+	int ret;
+
+	ret = spin_is_locked(&rq->lock);
+	put_cpu();
+	return ret;
+}
+
 /*
  * Debugging: various feature bits
  */

Index: linux-mcount.git/include/linux/linkage.h
===================================================================
@@ -3,6 +3,8 @@
 
 #include <asm/linkage.h>
 
+#define notrace __attribute__((no_instrument_function))
+
 #ifdef __cplusplus
 #define CPP_ASMLINKAGE extern "C"
 #else

Index: linux-mcount.git/include/linux/mcount.h
===================================================================
@@ -0,0 +1,38 @@
+#ifndef _LINUX_MCOUNT_H
+#define _LINUX_MCOUNT_H
+
+#ifdef CONFIG_MCOUNT
+extern int mcount_enabled;
+
+#include <linux/linkage.h>
+
+#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
+#define CALLER_ADDR1 ((unsigned long)__builtin_return_address(1))
+#define CALLER_ADDR2 ((unsigned long)__builtin_return_address(2))
+
+typedef void (*mcount_func_t)(unsigned long ip, unsigned long parent_ip);
+
+struct mcount_ops {
+	mcount_func_t func;
+	struct mcount_ops *next;
+};
+
+/*
+ * The mcount_ops must be a static and should also
+ * be read_mostly.  These functions do modify read_mostly variables
+ * so use them sparely. Never free an mcount_op or modify the
+ * next pointer after it has been registered. Even after unregistering
+ * it, the next pointer may still be used internally.
+ */
+int register_mcount_function(struct mcount_ops *ops);
+int unregister_mcount_function(struct mcount_ops *ops);
+void clear_mcount_function(void);
+
+extern void mcount(void);
+
+#else /* !CONFIG_MCOUNT */
+# define register_mcount_function(ops) do { } while (0)
+# define unregister_mcount_function(ops) do { } while (0)
+# define clear_mcount_function(ops) do { } while (0)
+#endif /* CONFIG_MCOUNT */
+#endif /* _LINUX_MCOUNT_H */

Index: linux-mcount.git/kernel/sysctl.c
===================================================================
@@ -46,6 +46,7 @@
 #include <linux/nfs_fs.h>
 #include <linux/acpi.h>
 #include <linux/reboot.h>
+#include <linux/mcount.h>
 
 #include <asm/uaccess.h>
 #include <asm/processor.h>

@@ -514,6 +515,16 @@ static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= &proc_dointvec,
 	},
+#ifdef CONFIG_MCOUNT
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "mcount_enabled",
+		.data		= &mcount_enabled,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+	},
+#endif
 #ifdef CONFIG_KMOD
 	{
 		.ctl_name	= KERN_MODPROBE,

151 lines not on this page · raw message

Index: linux-mcount.git/lib/smp_processor_id.c
===================================================================
@@ -7,7 +7,7 @@
 #include <linux/kallsyms.h>
 #include <linux/sched.h>
 
-unsigned int debug_smp_processor_id(void)
+notrace unsigned int debug_smp_processor_id(void)
 {
 	unsigned long preempt_count = preempt_count();
 	int this_cpu = raw_smp_processor_id();

Index: linux-mcount.git/arch/x86/kernel/head64.c
===================================================================
@@ -46,7 +46,7 @@ static void __init copy_bootdata(char *r
 	}
 }
 
-void __init x86_64_start_kernel(char * real_mode_data)
+notrace void __init x86_64_start_kernel(char *real_mode_data)
 {
 	int i;
 

Index: linux-mcount.git/arch/x86/kernel/setup64.c
===================================================================
@@ -114,7 +114,7 @@ void __init setup_per_cpu_areas(void)
 	}
 } 
 
-void pda_init(int cpu)
+notrace void pda_init(int cpu)
 { 
 	struct x8664_pda *pda = cpu_pda(cpu);
 

@@ -197,7 +197,7 @@ DEFINE_PER_CPU(struct orig_ist, orig_ist
  * 'CPU state barrier', nothing should get across.
  * A lot of state is already set up in PDA init.
  */
-void __cpuinit cpu_init (void)
+notrace void __cpuinit cpu_init(void)
 {
 	int cpu = stack_smp_processor_id();
 	struct tss_struct *t = &per_cpu(init_tss, cpu);

Index: linux-mcount.git/arch/x86/kernel/smpboot_64.c
===================================================================
@@ -317,7 +317,7 @@ static inline void set_cpu_sibling_map(i
 /*
  * Setup code on secondary processor (after comming out of the trampoline)
  */
-void __cpuinit start_secondary(void)
+notrace __cpuinit void start_secondary(void)
 {
 	/*
 	 * Dont put anything before smp_callin(), SMP

Index: linux-mcount.git/arch/x86/kernel/vsyscall_64.c
===================================================================
@@ -42,7 +42,8 @@
 #include <asm/topology.h>
 #include <asm/vgtod.h>
 
-#define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
+#define __vsyscall(nr) \
+		__attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
 #define __syscall_clobber "r11","rcx","memory"
 #define __pa_vsymbol(x)			\
 	({unsigned long v;  		\

Index: linux-mcount.git/arch/x86/vdso/vclock_gettime.c
===================================================================
@@ -24,7 +24,7 @@
 
 #define gtod vdso_vsyscall_gtod_data
 
-static long vdso_fallback_gettime(long clock, struct timespec *ts)
+notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
 {
 	long ret;
 	asm("syscall" : "=a" (ret) :

@@ -32,7 +32,7 @@ static long vdso_fallback_gettime(long c
 	return ret;
 }
 
-static inline long vgetns(void)
+notrace static inline long vgetns(void)
 {
 	long v;
 	cycles_t (*vread)(void);

@@ -41,7 +41,7 @@ static inline long vgetns(void)
 	return (v * gtod->clock.mult) >> gtod->clock.shift;
 }
 
-static noinline int do_realtime(struct timespec *ts)
+notrace static noinline int do_realtime(struct timespec *ts)
 {
 	unsigned long seq, ns;
 	do {

@@ -55,7 +55,8 @@ static noinline int do_realtime(struct t
 }
 
 /* Copy of the version in kernel/time.c which we cannot directly access */
-static void vset_normalized_timespec(struct timespec *ts, long sec, long nsec)
+notrace static void
+vset_normalized_timespec(struct timespec *ts, long sec, long nsec)
 {
 	while (nsec >= NSEC_PER_SEC) {
 		nsec -= NSEC_PER_SEC;

@@ -69,7 +70,7 @@ static void vset_normalized_timespec(str
 	ts->tv_nsec = nsec;
 }
 
-static noinline int do_monotonic(struct timespec *ts)
+notrace static noinline int do_monotonic(struct timespec *ts)
 {
 	unsigned long seq, ns, secs;
 	do {

@@ -83,7 +84,7 @@ static noinline int do_monotonic(struct 
 	return 0;
 }
 
-int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
+notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 {
 	if (likely(gtod->sysctl_enabled && gtod->clock.vread))
 		switch (clock) {

@@ -97,7 +98,7 @@ int __vdso_clock_gettime(clockid_t clock
 int clock_gettime(clockid_t, struct timespec *)
 	__attribute__((weak, alias("__vdso_clock_gettime")));
 
-int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
+notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 {
 	long ret;
 	if (likely(gtod->sysctl_enabled && gtod->clock.vread)) {

Index: linux-mcount.git/arch/x86/vdso/vgetcpu.c
===================================================================
@@ -13,7 +13,8 @@
 #include <asm/vgtod.h>
 #include "vextern.h"
 
-long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused)
+notrace long
+__vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused)
 {
 	unsigned int dummy, p;
 

Index: linux-mcount.git/include/asm-x86/vsyscall.h
===================================================================
@@ -24,7 +24,8 @@ enum vsyscall_num {
 	((unused, __section__ (".vsyscall_gtod_data"),aligned(16)))
 #define __section_vsyscall_clock __attribute__ \
 	((unused, __section__ (".vsyscall_clock"),aligned(16)))
-#define __vsyscall_fn __attribute__ ((unused,__section__(".vsyscall_fn")))
+#define __vsyscall_fn \
+	__attribute__ ((unused, __section__(".vsyscall_fn"))) notrace
 
 #define VGETCPU_RDTSCP	1
 #define VGETCPU_LSL	2

Index: linux-mcount.git/arch/x86/kernel/vsyscall_64.c
===================================================================
@@ -86,6 +86,7 @@ void update_vsyscall(struct timespec *wa
 	vsyscall_gtod_data.clock.mask = clock->mask;
 	vsyscall_gtod_data.clock.mult = clock->mult;
 	vsyscall_gtod_data.clock.shift = clock->shift;
+	vsyscall_gtod_data.clock.cycle_accumulated = clock->cycle_accumulated;
 	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
 	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
 	vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;

@@ -121,7 +122,7 @@ static __always_inline long time_syscall
 
 static __always_inline void do_vgettimeofday(struct timeval * tv)
 {
-	cycle_t now, base, mask, cycle_delta;
+	cycle_t now, base, accumulated, mask, cycle_delta;
 	unsigned seq;
 	unsigned long mult, shift, nsec;
 	cycle_t (*vread)(void);

@@ -135,6 +136,7 @@ static __always_inline void do_vgettimeo
 		}
 		now = vread();
 		base = __vsyscall_gtod_data.clock.cycle_last;
+		accumulated  = __vsyscall_gtod_data.clock.cycle_accumulated;
 		mask = __vsyscall_gtod_data.clock.mask;
 		mult = __vsyscall_gtod_data.clock.mult;
 		shift = __vsyscall_gtod_data.clock.shift;

@@ -145,6 +147,7 @@ static __always_inline void do_vgettimeo
 
 	/* calculate interval: */
 	cycle_delta = (now - base) & mask;
+	cycle_delta += accumulated;
 	/* convert to nsecs: */
 	nsec += (cycle_delta * mult) >> shift;
 

Index: linux-mcount.git/include/asm-x86/vgtod.h
===================================================================
@@ -15,7 +15,7 @@ struct vsyscall_gtod_data {
 	struct timezone sys_tz;
 	struct { /* extract of a clocksource struct */
 		cycle_t (*vread)(void);
-		cycle_t	cycle_last;
+		cycle_t	cycle_last, cycle_accumulated;
 		cycle_t	mask;
 		u32	mult;
 		u32	shift;

95 lines not on this page · raw message

Index: linux-mcount.git/kernel/time/timekeeping.c
===================================================================
@@ -66,16 +66,10 @@ static struct clocksource *clock; /* poi
  */
 static inline s64 __get_nsec_offset(void)
 {
-	cycle_t cycle_now, cycle_delta;
+	cycle_t cycle_delta;
 	s64 ns_offset;
 
-	/* read clocksource: */
-	cycle_now = clocksource_read(clock);
-
-	/* calculate the delta since the last update_wall_time: */
-	cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
-
-	/* convert to nanoseconds: */
+	cycle_delta = clocksource_get_cycles(clock, clocksource_read(clock));
 	ns_offset = cyc2ns(clock, cycle_delta);
 
 	return ns_offset;

@@ -195,7 +189,7 @@ static void change_clocksource(void)
 
 	clock = new;
 	clock->cycle_last = now;
-
+	clock->cycle_accumulated = 0;
 	clock->error = 0;
 	clock->xtime_nsec = 0;
 	clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH);

@@ -205,9 +199,15 @@ static void change_clocksource(void)
 	printk(KERN_INFO "Time: %s clocksource has been installed.\n",
 	       clock->name);
 }
+
+void timekeeping_accumulate(void)
+{
+	clocksource_accumulate(clock, clocksource_read(clock));
+}
 #else
 static inline void change_clocksource(void) { }
 static inline s64 __get_nsec_offset(void) { return 0; }
+void timekeeping_accumulate(void) { }
 #endif
 
 /**

@@ -302,6 +302,7 @@ static int timekeeping_resume(struct sys
 	timespec_add_ns(&xtime, timekeeping_suspend_nsecs);
 	/* re-base the last cycle value */
 	clock->cycle_last = clocksource_read(clock);
+	clock->cycle_accumulated = 0;
 	clock->error = 0;
 	timekeeping_suspended = 0;
 	write_sequnlock_irqrestore(&xtime_lock, flags);

@@ -448,27 +449,28 @@ static void clocksource_adjust(s64 offse
  */
 void update_wall_time(void)
 {
-	cycle_t offset;
+	cycle_t cycle_now;
 
 	/* Make sure we're fully resumed: */
 	if (unlikely(timekeeping_suspended))
 		return;
 
 #ifdef CONFIG_GENERIC_TIME
-	offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask;
+	cycle_now = clocksource_read(clock);
 #else
-	offset = clock->cycle_interval;
+	cycle_now = clock->cycle_last + clock->cycle_interval;
 #endif
+	clocksource_accumulate(clock, cycle_now);
+
 	clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift;
 
 	/* normally this loop will run just once, however in the
 	 * case of lost or late ticks, it will accumulate correctly.
 	 */
-	while (offset >= clock->cycle_interval) {
+	while (clock->cycle_accumulated >= clock->cycle_interval) {
 		/* accumulate one interval */
 		clock->xtime_nsec += clock->xtime_interval;
-		clock->cycle_last += clock->cycle_interval;
-		offset -= clock->cycle_interval;
+		clock->cycle_accumulated -= clock->cycle_interval;
 
 		if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) {
 			clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift;

@@ -482,13 +484,13 @@ void update_wall_time(void)
 	}
 
 	/* correct the clock when NTP error is too big */
-	clocksource_adjust(offset);
+	clocksource_adjust(clock->cycle_accumulated);
 
 	/* store full nanoseconds into xtime */
 	xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift;
 	clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift;
 
-	update_xtime_cache(cyc2ns(clock, offset));
+	update_xtime_cache(cyc2ns(clock, clock->cycle_accumulated));
 
 	/* check to see if there is a new clocksource to use */
 	change_clocksource();

Index: linux-mcount.git/arch/powerpc/kernel/time.c
===================================================================
@@ -773,7 +773,8 @@ void update_vsyscall(struct timespec *wa
 	stamp_xsec = (u64) xtime.tv_nsec * XSEC_PER_SEC;
 	do_div(stamp_xsec, 1000000000);
 	stamp_xsec += (u64) xtime.tv_sec * XSEC_PER_SEC;
-	update_gtod(clock->cycle_last, stamp_xsec, t2x);
+	update_gtod(clock->cycle_last-clock->cycle_accumulated,
+		    stamp_xsec, t2x);
 }
 
 void update_vsyscall_tz(void)

Index: linux-2.6-lttng/ltt/ltt-timestamp.c
===================================================================
@@ -0,0 +1,210 @@
+/*
+ * (C) Copyright	2006,2007 -
+ * 		Mathieu Desnoyers ([email protected])
+ *
+ * notes : ltt-timestamp timer-based clock cannot be used for early tracing in
+ * the boot process, as it depends on timer interrupts.
+ *
+ * The timer is only on one CPU to support hotplug.
+ * We have the choice between schedule_delayed_work_on and an IPI to get each
+ * CPU to write the heartbeat. IPI has been chosen because it is considered
+ * faster than passing through the timer to get the work scheduled on all the
+ * CPUs.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+#include <linux/cpu.h>
+#include <linux/timex.h>
+#include <linux/bitops.h>
+#include <linux/ltt.h>
+#include <linux/smp.h>
+#include <linux/sched.h> /* FIX for m68k local_irq_enable in on_each_cpu */
+
+/*
+ * Number of hardware clock bits. The higher order bits are expected to be 0.
+ * If the hardware clock source has more than 32 bits, the bits higher than the
+ * 32nd will be truncated by a cast to a 32 bits unsigned. Range : 1 - 32.
+ * (too few bits would be unrealistic though, since we depend on the timer to
+ * detect the overflows).
+ */
+#define HW_BITS				32
+
+#define HW_BITMASK			((1ULL << HW_BITS) - 1)
+#define HW_LSB(hw)			((hw) & HW_BITMASK)
+#define SW_MSB(sw)			((sw) & ~HW_BITMASK)
+
+/* Expected maximum interrupt latency in ms : 15ms, *2 for security */
+#define EXPECTED_INTERRUPT_LATENCY	30
+
+atomic_t lttng_generic_clock;
+EXPORT_SYMBOL(lttng_generic_clock);
+
+static struct timer_list stsc_timer;
+static unsigned int precalc_expire;
+
+static struct synthetic_tsc_struct {
+	union {
+		u64 val;
+		struct {
+#ifdef __BIG_ENDIAN
+			u32 msb;
+			u32 lsb;
+#else
+			u32 lsb;
+			u32 msb;
+#endif
+		} sel;
+	} tsc[2];
+	unsigned int index;	/* Index of the current synth. tsc. */
+} ____cacheline_aligned synthetic_tsc[NR_CPUS];
+
+/* Called from IPI : either in interrupt or process context */
+static void ltt_update_synthetic_tsc(void)
+{
+	struct synthetic_tsc_struct *cpu_synth;
+	u32 tsc;
+
+	preempt_disable();
+	cpu_synth = &synthetic_tsc[smp_processor_id()];
+	tsc = ltt_get_timestamp32();		/* Hardware clocksource read */
+
+	if (tsc < HW_LSB(cpu_synth->tsc[cpu_synth->index].sel.lsb)) {
+		unsigned int new_index = 1 - cpu_synth->index; /* 0 <-> 1 */
+		/*
+		 * Overflow
+		 * Non atomic update of the non current synthetic TSC, followed
+		 * by an atomic index change. There is no write concurrency,
+		 * so the index read/write does not need to be atomic.
+		 */
+		cpu_synth->tsc[new_index].val =
+			(SW_MSB(cpu_synth->tsc[cpu_synth->index].val)
+				| (u64)tsc) + (1ULL << HW_BITS);
+		cpu_synth->index = new_index;	/* atomic change of index */
+	} else {
+		/*
+		 * No overflow : We know that the only bits changed are
+		 * contained in the 32 LSBs, which can be written to atomically.
+		 */
+		cpu_synth->tsc[cpu_synth->index].sel.lsb =
+			SW_MSB(cpu_synth->tsc[cpu_synth->index].sel.lsb) | tsc;
+	}
+	preempt_enable();
+}
+
+/* Called from buffer switch : in _any_ context (even NMI) */
+u64 ltt_read_synthetic_tsc(void)
+{
+	struct synthetic_tsc_struct *cpu_synth;
+	u64 ret;
+	unsigned int index;
+	u32 tsc;
+
+	preempt_disable();
+	cpu_synth = &synthetic_tsc[smp_processor_id()];
+	index = cpu_synth->index;		/* atomic read */
+	tsc = ltt_get_timestamp32();		/* Hardware clocksource read */
+
+	/* Overflow detection */
+	if (unlikely(tsc < HW_LSB(cpu_synth->tsc[index].sel.lsb)))
+		ret = (SW_MSB(cpu_synth->tsc[index].val) | (u64)tsc)
+			+ (1ULL << HW_BITS);
+	else
+		ret = SW_MSB(cpu_synth->tsc[index].val) | (u64)tsc;
+	preempt_enable();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ltt_read_synthetic_tsc);
+
+static void synthetic_tsc_ipi(void *info)
+{
+	ltt_update_synthetic_tsc();
+}
+
+/* We need to be in process context to do an IPI */
+static void synthetic_tsc_work(struct work_struct *work)
+{
+	on_each_cpu(synthetic_tsc_ipi, NULL, 1, 1);
+}
+static DECLARE_WORK(stsc_work, synthetic_tsc_work);
+
+/*
+ * stsc_timer : - Timer function synchronizing synthetic TSC.
+ * @data: unused
+ *
+ * Guarantees at least 1 execution before low word of TSC wraps.
+ */
+static void stsc_timer_fct(unsigned long data)
+{
+	PREPARE_WORK(&stsc_work, synthetic_tsc_work);
+	schedule_work(&stsc_work);
+
+	mod_timer(&stsc_timer, jiffies + precalc_expire);
+}
+
+/*
+ * precalc_stsc_interval: - Precalculates the interval between the clock
+ * wraparounds.
+ */
+static int __init precalc_stsc_interval(void)
+{
+	precalc_expire =
+		(HW_BITMASK / ((ltt_frequency() / HZ * ltt_freq_scale()) << 1)
+			- 1 - (EXPECTED_INTERRUPT_LATENCY * HZ / 1000)) >> 1;
+	WARN_ON(precalc_expire == 0);
+	printk(KERN_DEBUG "Synthetic TSC timer will fire each %u jiffies.\n",
+		precalc_expire);
+	return 0;
+}
+
+/*
+ * 	hotcpu_callback - CPU hotplug callback
+ * 	@nb: notifier block
+ * 	@action: hotplug action to take
+ * 	@hcpu: CPU number
+ *
+ *	Sets the new CPU's current synthetic TSC to the same value as the
+ *	currently running CPU.
+ *
+ * 	Returns the success/failure of the operation. (NOTIFY_OK, NOTIFY_BAD)
+ */
+static int __cpuinit hotcpu_callback(struct notifier_block *nb,
+				unsigned long action,
+				void *hcpu)
+{
+	unsigned int hotcpu = (unsigned long)hcpu;
+	struct synthetic_tsc_struct *cpu_synth;
+	u64 local_count;
+
+	switch (action) {
+	case CPU_UP_PREPARE:
+		cpu_synth = &synthetic_tsc[hotcpu];
+		local_count = ltt_read_synthetic_tsc();
+		cpu_synth->tsc[0].val = local_count;
+		cpu_synth->index = 0;
+		smp_wmb();	/* Writing in data of CPU about to come up */
+		break;
+	case CPU_ONLINE:
+		/* As we are preemptible, make sure it runs on the right cpu */
+		smp_call_function_single(hotcpu, synthetic_tsc_ipi, NULL, 1, 0);
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+/* Called from one CPU, before any tracing starts, to init each structure */
+static int __init ltt_init_synthetic_tsc(void)
+{
+	hotcpu_notifier(hotcpu_callback, 3);
+	precalc_stsc_interval();
+	init_timer(&stsc_timer);
+	stsc_timer.function = stsc_timer_fct;
+	stsc_timer.expires = jiffies + precalc_expire;
+	add_timer(&stsc_timer);
+	return 0;
+}
+
+__initcall(ltt_init_synthetic_tsc);

Index: linux-mcount.git/include/linux/clocksource.h
===================================================================
@@ -273,6 +273,9 @@ extern struct clocksource* clocksource_g
 extern void clocksource_change_rating(struct clocksource *cs, int rating);
 extern void clocksource_resume(void);
 
+/* used to initialize clock */
+extern struct clocksource clocksource_jiffies;
+
 #ifdef CONFIG_GENERIC_TIME_VSYSCALL
 extern void update_vsyscall(struct timespec *ts, struct clocksource *c);
 extern void update_vsyscall_tz(void);

Index: linux-mcount.git/kernel/time/timekeeping.c
===================================================================
@@ -53,8 +53,13 @@ static inline void update_xtime_cache(u6
 	timespec_add_ns(&xtime_cache, nsec);
 }
 
-static struct clocksource *clock; /* pointer to current clocksource */
-
+/*
+ * pointer to current clocksource
+ *  Just in case we use clocksource_read before we initialize
+ *  the actual clock source. Instead of calling a NULL read pointer
+ *  we return jiffies.
+ */
+static struct clocksource *clock = &clocksource_jiffies;
 
 #ifdef CONFIG_GENERIC_TIME
 /**

Index: linux-mcount.git/include/linux/clocksource.h
===================================================================
@@ -88,8 +88,16 @@ struct clocksource {
 	 */
 	struct {
 		cycle_t cycle_last, cycle_accumulated;
-	} ____cacheline_aligned_in_smp;
 
+		/* base structure provides lock-free read
+		 * access to a virtualized 64bit counter
+		 * Uses RCU-like update.
+		 */
+		struct {
+			cycle_t cycle_base_last, cycle_base;
+		} base[2];
+		int base_num;
+	} ____cacheline_aligned_in_smp;
 	u64 xtime_nsec;
 	s64 error;
 

@@ -175,19 +183,30 @@ static inline cycle_t clocksource_read(s
 }
 
 /**
- * clocksource_get_cycles: - Access the clocksource's accumulated cycle value
+ * clocksource_get_basecycles: - get the clocksource's accumulated cycle value
  * @cs:		pointer to clocksource being read
  * @now:	current cycle value
  *
  * Uses the clocksource to return the current cycle_t value.
  * NOTE!!!: This is different from clocksource_read, because it
- * returns the accumulated cycle value! Must hold xtime lock!
+ * returns a 64bit wide accumulated value.
  */
 static inline cycle_t
-clocksource_get_cycles(struct clocksource *cs, cycle_t now)
+clocksource_get_basecycles(struct clocksource *cs)
 {
-	cycle_t offset = (now - cs->cycle_last) & cs->mask;
-	offset += cs->cycle_accumulated;
+	int num;
+	cycle_t now, offset;
+
+	preempt_disable();
+	num = cs->base_num;
+	/* base_num is shared, and some archs are wacky */
+	smp_read_barrier_depends();
+	now = clocksource_read(cs);
+	offset = (now - cs->base[num].cycle_base_last);
+	offset &= cs->mask;
+	offset += cs->base[num].cycle_base;
+	preempt_enable();
+
 	return offset;
 }
 

@@ -197,11 +216,27 @@ clocksource_get_cycles(struct clocksourc
  * @now:	current cycle value
  *
  * Used to avoids clocksource hardware overflow by periodically
- * accumulating the current cycle delta. Must hold xtime write lock!
+ * accumulating the current cycle delta. Uses RCU-like update, but
+ * ***still requires the xtime_lock is held for writing!***
  */
 static inline void clocksource_accumulate(struct clocksource *cs, cycle_t now)
 {
-	cycle_t offset = (now - cs->cycle_last) & cs->mask;
+	/*
+	 * First update the monotonic base portion.
+	 * The dual array update method allows for lock-free reading.
+	 * 'num' is always 1 or 0.
+	 */
+	int num = 1 - cs->base_num;
+	cycle_t offset = (now - cs->base[1-num].cycle_base_last);
+	offset &= cs->mask;
+	cs->base[num].cycle_base = cs->base[1-num].cycle_base + offset;
+	cs->base[num].cycle_base_last = now;
+	/* make sure this array is visible to the world first */
+	smp_wmb();
+	cs->base_num = num;
+
+	/* Now update the cycle_accumulated portion */
+	offset = (now - cs->cycle_last) & cs->mask;
 	cs->cycle_last = now;
 	cs->cycle_accumulated += offset;
 }

@@ -272,6 +307,9 @@ extern int clocksource_register(struct c
 extern struct clocksource* clocksource_get_next(void);
 extern void clocksource_change_rating(struct clocksource *cs, int rating);
 extern void clocksource_resume(void);
+extern cycle_t get_monotonic_cycles(void);
+extern unsigned long cycles_to_usecs(cycle_t cycles);
+extern cycle_t usecs_to_cycles(unsigned long usecs);
 
 /* used to initialize clock */
 extern struct clocksource clocksource_jiffies;

Index: linux-mcount.git/kernel/time/timekeeping.c
===================================================================
@@ -71,10 +71,12 @@ static struct clocksource *clock = &cloc
  */
 static inline s64 __get_nsec_offset(void)
 {
-	cycle_t cycle_delta;
+	cycle_t now, cycle_delta;
 	s64 ns_offset;
 
-	cycle_delta = clocksource_get_cycles(clock, clocksource_read(clock));
+	now = clocksource_read(clock);
+	cycle_delta = (now - clock->cycle_last) & clock->mask;
+	cycle_delta += clock->cycle_accumulated;
 	ns_offset = cyc2ns(clock, cycle_delta);
 
 	return ns_offset;

@@ -103,6 +105,26 @@ static inline void __get_realtime_clock_
 	timespec_add_ns(ts, nsecs);
 }
 
+cycle_t notrace get_monotonic_cycles(void)
+{
+	return clocksource_get_basecycles(clock);
+}
+
+unsigned long notrace cycles_to_usecs(cycle_t cycles)
+{
+	u64 ret = cyc2ns(clock, cycles);
+
+	ret += NSEC_PER_USEC/2; /* For rounding in do_div() */
+	do_div(ret, NSEC_PER_USEC);
+
+	return ret;
+}
+
+cycle_t notrace usecs_to_cycles(unsigned long usecs)
+{
+	return ns2cyc(clock, (u64)usecs * 1000);
+}
+
 /**
  * getnstimeofday - Returns the time of day in a timespec
  * @ts:		pointer to the timespec to be set

Index: linux-mcount.git/arch/x86/kernel/apic_32.c
===================================================================
@@ -577,7 +577,7 @@ static void local_apic_timer_interrupt(v
  *   interrupt as well. Thus we cannot inline the local irq ... ]
  */
 
-void fastcall smp_apic_timer_interrupt(struct pt_regs *regs)
+notrace fastcall void smp_apic_timer_interrupt(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 

Index: linux-mcount.git/arch/x86/kernel/hpet.c
===================================================================
@@ -295,7 +295,7 @@ static int hpet_legacy_next_event(unsign
 /*
  * Clock source related code
  */
-static cycle_t read_hpet(void)
+static notrace cycle_t read_hpet(void)
 {
 	return (cycle_t)hpet_readl(HPET_COUNTER);
 }

Index: linux-mcount.git/arch/x86/kernel/time_32.c
===================================================================
@@ -122,7 +122,7 @@ static int set_rtc_mmss(unsigned long no
 
 int timer_ack;
 
-unsigned long profile_pc(struct pt_regs *regs)
+notrace unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 

Index: linux-mcount.git/arch/x86/kernel/tsc_32.c
===================================================================
@@ -269,7 +269,7 @@ core_initcall(cpufreq_tsc);
 
 static unsigned long current_tsc_khz = 0;
 
-static cycle_t read_tsc(void)
+static notrace cycle_t read_tsc(void)
 {
 	cycle_t ret;
 

Index: linux-mcount.git/arch/x86/kernel/tsc_64.c
===================================================================
@@ -248,13 +248,13 @@ __setup("notsc", notsc_setup);
 
 
 /* clock source code: */
-static cycle_t read_tsc(void)
+static notrace cycle_t read_tsc(void)
 {
 	cycle_t ret = (cycle_t)get_cycles_sync();
 	return ret;
 }
 
-static cycle_t __vsyscall_fn vread_tsc(void)
+static notrace cycle_t __vsyscall_fn vread_tsc(void)
 {
 	cycle_t ret = (cycle_t)get_cycles_sync();
 	return ret;

Index: linux-mcount.git/arch/x86/lib/delay_32.c
===================================================================
@@ -24,7 +24,7 @@
 #endif
 
 /* simple loop based delay: */
-static void delay_loop(unsigned long loops)
+static notrace void delay_loop(unsigned long loops)
 {
 	int d0;
 

@@ -39,7 +39,7 @@ static void delay_loop(unsigned long loo
 }
 
 /* TSC based delay: */
-static void delay_tsc(unsigned long loops)
+static notrace void delay_tsc(unsigned long loops)
 {
 	unsigned long bclock, now;
 

@@ -72,7 +72,7 @@ int read_current_timer(unsigned long *ti
 	return -1;
 }
 
-void __delay(unsigned long loops)
+notrace void __delay(unsigned long loops)
 {
 	delay_fn(loops);
 }

Index: linux-mcount.git/drivers/clocksource/acpi_pm.c
===================================================================
@@ -30,13 +30,13 @@
  */
 u32 pmtmr_ioport __read_mostly;
 
-static inline u32 read_pmtmr(void)
+static inline notrace u32 read_pmtmr(void)
 {
 	/* mask the output to 24 bits */
 	return inl(pmtmr_ioport) & ACPI_PM_MASK;
 }
 
-u32 acpi_pm_read_verified(void)
+notrace u32 acpi_pm_read_verified(void)
 {
 	u32 v1 = 0, v2 = 0, v3 = 0;
 

@@ -56,12 +56,12 @@ u32 acpi_pm_read_verified(void)
 	return v2;
 }
 
-static cycle_t acpi_pm_read_slow(void)
+static notrace cycle_t acpi_pm_read_slow(void)
 {
 	return (cycle_t)acpi_pm_read_verified();
 }
 
-static cycle_t acpi_pm_read(void)
+static notrace cycle_t acpi_pm_read(void)
 {
 	return (cycle_t)read_pmtmr();
 }

Index: linux-mcount.git/include/linux/clocksource.h
===================================================================
@@ -197,7 +197,8 @@ clocksource_get_basecycles(struct clocks
 	int num;
 	cycle_t now, offset;
 
-	preempt_disable();
+	/* This code is used for tracing. */
+	preempt_disable_notrace();
 	num = cs->base_num;
 	/* base_num is shared, and some archs are wacky */
 	smp_read_barrier_depends();

@@ -205,7 +206,7 @@ clocksource_get_basecycles(struct clocks
 	offset = (now - cs->base[num].cycle_base_last);
 	offset &= cs->mask;
 	offset += cs->base[num].cycle_base;
-	preempt_enable();
+	preempt_enable_notrace();
 
 	return offset;
 }

Index: linux-mcount.git/include/linux/preempt.h
===================================================================
@@ -52,6 +52,34 @@ do { \
 	preempt_check_resched(); \
 } while (0)
 
+/* For debugging and tracer internals only! */
+#define add_preempt_count_notrace(val)			\
+	do { preempt_count() += (val); } while (0)
+#define sub_preempt_count_notrace(val)			\
+	do { preempt_count() -= (val); } while (0)
+#define inc_preempt_count_notrace() add_preempt_count_notrace(1)
+#define dec_preempt_count_notrace() sub_preempt_count_notrace(1)
+
+#define preempt_disable_notrace() \
+do { \
+	inc_preempt_count_notrace(); \
+	barrier(); \
+} while (0)
+
+#define preempt_enable_no_resched_notrace() \
+do { \
+	barrier(); \
+	dec_preempt_count_notrace(); \
+} while (0)
+
+/* preempt_check_resched is OK to trace */
+#define preempt_enable_notrace() \
+do { \
+	preempt_enable_no_resched_notrace(); \
+	barrier(); \
+	preempt_check_resched(); \
+} while (0)
+
 #else
 
 #define preempt_disable()		do { } while (0)

@@ -59,6 +87,10 @@ do { \
 #define preempt_enable()		do { } while (0)
 #define preempt_check_resched()		do { } while (0)
 
+#define preempt_disable_notrace()		do { } while (0)
+#define preempt_enable_no_resched_notrace()	do { } while (0)
+#define preempt_enable_notrace()		do { } while (0)
+
 #endif
 
 #ifdef CONFIG_PREEMPT_NOTIFIERS

1,165 lines not on this page · raw message

96 lines not on this page · raw message

72 lines not on this page · raw message

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -2198,6 +2198,8 @@ context_switch(struct rq *rq, struct tas
 	struct mm_struct *mm, *oldmm;
 
 	prepare_task_switch(rq, prev, next);
+	trace_mark(kernel_sched_schedule,
+		   "prev %p next %p", prev, next);
 	mm = next->mm;
 	oldmm = prev->active_mm;
 	/*

Index: linux-mcount.git/include/linux/sched.h
===================================================================
@@ -2055,6 +2055,8 @@ static inline void migration_init(void)
 }
 #endif
 
+#define TASK_STATE_TO_CHAR_STR "RSDTtZX"
+
 #endif /* __KERNEL__ */
 
 #endif

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -5149,7 +5149,7 @@ out_unlock:
 	return retval;
 }
 
-static const char stat_nam[] = "RSDTtZX";
+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 
 void sched_show_task(struct task_struct *p)
 {

166 lines not on this page · raw message

85 lines not on this page · raw message

Index: linux-mcount.git/lib/tracing/tracer.h
===================================================================
@@ -10,6 +10,14 @@ struct tracing_function {
 	unsigned long parent_ip;
 };
 
+struct tracing_sched_switch {
+	unsigned int prev_pid;
+	unsigned char prev_prio;
+	unsigned char prev_state;
+	unsigned int next_pid;
+	unsigned char next_prio;
+};
+
 struct tracing_entry {
 	char type;
 	char cpu;  /* who will want to trace more than 256 CPUS? */

@@ -18,7 +26,10 @@ struct tracing_entry {
 	int pid;
 	cycle_t t;
 	char comm[TASK_COMM_LEN];
-	struct tracing_function fn;
+	union {
+		struct tracing_function fn;
+		struct tracing_sched_switch ctx;
+	};
 };
 
 struct tracing_trace_cpu {

@@ -82,11 +93,21 @@ void tracing_function_trace(struct traci
 			    unsigned long ip,
 			    unsigned long parent_ip,
 			    unsigned long flags);
+void tracing_sched_switch_trace(struct tracing_trace *tr,
+				struct tracing_trace_cpu *data,
+				struct task_struct *prev,
+				struct task_struct *next,
+				unsigned long flags);
+
 
 void tracing_start_function_trace(void);
 void tracing_stop_function_trace(void);
 int register_trace(struct trace_types_struct *type);
 void unregister_trace(struct trace_types_struct *type);
+void tracing_start_sched_switch(void);
+void tracing_stop_sched_switch(void);
+
+extern int tracing_sched_switch_enabled;
 
 static inline notrace cycle_t now(void)
 {

Index: linux-mcount.git/lib/tracing/trace_function.c
===================================================================
@@ -29,10 +29,12 @@ static notrace void start_function_trace
 {
 	function_reset(tr);
 	tracing_start_function_trace();
+	tracing_start_sched_switch();
 }
 
 static notrace void stop_function_trace(struct tracing_trace *tr)
 {
+	tracing_stop_sched_switch();
 	tracing_stop_function_trace();
 }
 

Index: linux-mcount.git/lib/tracing/trace_sched_switch.c
===================================================================
@@ -33,6 +33,11 @@ static notrace void sched_switch_callbac
 	va_list ap;
 	int cpu;
 
+	if (!atomic_read(&sched_ref))
+		return;
+
+	tracing_record_cmdline(current);
+
 	if (!trace_enabled)
 		return;
 

183 lines not on this page · raw message

Index: linux-mcount.git/lib/tracing/tracer.h
===================================================================
@@ -25,7 +25,6 @@ struct tracing_entry {
 	char preempt_count; /* assumes PREEMPT_MASK is 8 bits or less */
 	int pid;
 	cycle_t t;
-	char comm[TASK_COMM_LEN];
 	union {
 		struct tracing_function fn;
 		struct tracing_sched_switch ctx;

@@ -98,7 +97,7 @@ void tracing_sched_switch_trace(struct t
 				struct task_struct *prev,
 				struct task_struct *next,
 				unsigned long flags);
-
+void tracing_record_cmdline(struct task_struct *tsk);
 
 void tracing_start_function_trace(void);
 void tracing_stop_function_trace(void);

Index: linux-mcount.git/lib/tracing/tracer.h
===================================================================
@@ -113,4 +113,18 @@ static inline notrace cycle_t now(void)
 	return get_monotonic_cycles();
 }
 
+#ifdef CONFIG_CONTEXT_SWITCH_TRACER
+typedef void (*tracer_switch_func_t)(void *private,
+				     struct task_struct *prev,
+				     struct task_struct *next);
+struct tracer_switch_ops {
+	tracer_switch_func_t func;
+	void *private;
+	struct tracer_switch_ops *next;
+};
+
+extern int register_tracer_switch(struct tracer_switch_ops *ops);
+extern int unregister_tracer_switch(struct tracer_switch_ops *ops);
+#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
+
 #endif /* _LINUX_MCOUNT_TRACER_H */

157 lines not on this page · raw message

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -1885,6 +1885,10 @@ static int try_to_wake_up(struct task_st
 
 out_activate:
 #endif /* CONFIG_SMP */
+	trace_mark(kernel_sched_wakeup,
+		   "p %p rq->curr %p",
+		   p, rq->curr);
+
 	schedstat_inc(p, se.nr_wakeups);
 	if (sync)
 		schedstat_inc(p, se.nr_wakeups_sync);

@@ -2026,6 +2030,10 @@ void fastcall wake_up_new_task(struct ta
 		p->sched_class->task_new(rq, p);
 		inc_nr_running(p, rq);
 	}
+	trace_mark(kernel_sched_wakeup_new,
+		   "p %p rq->curr %p",
+		   p, rq->curr);
+
 	check_preempt_curr(rq, p);
 #ifdef CONFIG_SMP
 	if (p->sched_class->task_wake_up)

377 lines not on this page · raw message

183 lines not on this page · raw message

Index: linux-mcount.git/lib/tracing/tracer.h
===================================================================
@@ -69,6 +69,7 @@ struct trace_types_struct {
 	void (*stop)(struct tracing_iterator *iter);
 	void (*ctrl_update)(struct tracing_trace *tr);
 	struct trace_types_struct *next;
+	int print_max;
 };
 
 struct tracing_iterator {

@@ -107,6 +108,10 @@ void tracing_start_sched_switch(void);
 void tracing_stop_sched_switch(void);
 
 extern int tracing_sched_switch_enabled;
+extern unsigned long tracing_max_latency;
+extern unsigned long tracing_thresh;
+
+void update_max_tr(struct tracing_trace *tr, struct task_struct *tsk, int cpu);
 
 static inline notrace cycle_t now(void)
 {

Index: linux-mcount.git/arch/x86/kernel/process_64.c
===================================================================
@@ -233,7 +233,10 @@ void cpu_idle (void)
 			 */
 			local_irq_disable();
 			enter_idle();
+			/* Don't trace irqs off for idle */
+			stop_critical_timings();
 			idle();
+			start_critical_timings();
 			/* In many cases the interrupt that ended idle
 			   has already called exit_idle. But some idle
 			   loops can be woken up without interrupt. */

Index: linux-mcount.git/include/asm-x86/irqflags_32.h
===================================================================
@@ -139,9 +139,9 @@ static inline int raw_irqs_disabled(void
 static inline void trace_hardirqs_fixup_flags(unsigned long flags)
 {
 	if (raw_irqs_disabled_flags(flags))
-		trace_hardirqs_off();
+		__trace_hardirqs_off();
 	else
-		trace_hardirqs_on();
+		__trace_hardirqs_on();
 }
 
 static inline void trace_hardirqs_fixup(void)

Index: linux-mcount.git/include/asm-x86/irqflags_64.h
===================================================================
@@ -120,9 +120,9 @@ static inline int raw_irqs_disabled(void
 static inline void trace_hardirqs_fixup_flags(unsigned long flags)
 {
 	if (raw_irqs_disabled_flags(flags))
-		trace_hardirqs_off();
+		__trace_hardirqs_off();
 	else
-		trace_hardirqs_on();
+		__trace_hardirqs_on();
 }
 
 static inline void trace_hardirqs_fixup(void)

Index: linux-mcount.git/include/linux/irqflags.h
===================================================================
@@ -12,10 +12,21 @@
 #define _LINUX_TRACE_IRQFLAGS_H
 
 #ifdef CONFIG_TRACE_IRQFLAGS
-  extern void trace_hardirqs_on(void);
-  extern void trace_hardirqs_off(void);
+# include <linux/mcount.h>
+  extern void trace_hardirqs_on_caller(unsigned long ip);
+  extern void trace_hardirqs_off_caller(unsigned long ip);
   extern void trace_softirqs_on(unsigned long ip);
   extern void trace_softirqs_off(unsigned long ip);
+  extern void trace_hardirqs_on(void);
+  extern void trace_hardirqs_off(void);
+  static inline void notrace __trace_hardirqs_on(void)
+  {
+	trace_hardirqs_on_caller(CALLER_ADDR0);
+  }
+  static inline void notrace __trace_hardirqs_off(void)
+  {
+	trace_hardirqs_off_caller(CALLER_ADDR0);
+  }
 # define trace_hardirq_context(p)	((p)->hardirq_context)
 # define trace_softirq_context(p)	((p)->softirq_context)
 # define trace_hardirqs_enabled(p)	((p)->hardirqs_enabled)

@@ -28,6 +39,8 @@
 #else
 # define trace_hardirqs_on()		do { } while (0)
 # define trace_hardirqs_off()		do { } while (0)
+# define __trace_hardirqs_on()		do { } while (0)
+# define __trace_hardirqs_off()		do { } while (0)
 # define trace_softirqs_on(ip)		do { } while (0)
 # define trace_softirqs_off(ip)		do { } while (0)
 # define trace_hardirq_context(p)	0

@@ -41,24 +54,32 @@
 # define INIT_TRACE_IRQFLAGS
 #endif
 
+#ifdef CONFIG_CRITICAL_IRQSOFF_TIMING
+ extern void stop_critical_timings(void);
+ extern void start_critical_timings(void);
+#else
+# define stop_critical_timings() do { } while (0)
+# define start_critical_timings() do { } while (0)
+#endif
+
 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 
 #include <asm/irqflags.h>
 
 #define local_irq_enable() \
-	do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
+	do { __trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
 #define local_irq_disable() \
-	do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
+	do { raw_local_irq_disable(); __trace_hardirqs_off(); } while (0)
 #define local_irq_save(flags) \
-	do { raw_local_irq_save(flags); trace_hardirqs_off(); } while (0)
+	do { raw_local_irq_save(flags); __trace_hardirqs_off(); } while (0)
 
 #define local_irq_restore(flags)				\
 	do {							\
 		if (raw_irqs_disabled_flags(flags)) {		\
 			raw_local_irq_restore(flags);		\
-			trace_hardirqs_off();			\
+			__trace_hardirqs_off();			\
 		} else {					\
-			trace_hardirqs_on();			\
+			__trace_hardirqs_on();			\
 			raw_local_irq_restore(flags);		\
 		}						\
 	} while (0)

@@ -76,7 +97,7 @@
 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 #define safe_halt()						\
 	do {							\
-		trace_hardirqs_on();				\
+		__trace_hardirqs_on();				\
 		raw_safe_halt();				\
 	} while (0)
 

Index: linux-mcount.git/include/linux/mcount.h
===================================================================
@@ -6,10 +6,6 @@ extern int mcount_enabled;
 
 #include <linux/linkage.h>
 
-#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
-#define CALLER_ADDR1 ((unsigned long)__builtin_return_address(1))
-#define CALLER_ADDR2 ((unsigned long)__builtin_return_address(2))
-
 typedef void (*mcount_func_t)(unsigned long ip, unsigned long parent_ip);
 
 struct mcount_ops {

@@ -35,4 +31,31 @@ extern void mcount(void);
 # define unregister_mcount_function(ops) do { } while (0)
 # define clear_mcount_function(ops) do { } while (0)
 #endif /* CONFIG_MCOUNT */
+
+
+#ifdef CONFIG_FRAME_POINTER
+/* TODO: need to fix this for ARM */
+# define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
+# define CALLER_ADDR1 ((unsigned long)__builtin_return_address(1))
+# define CALLER_ADDR2 ((unsigned long)__builtin_return_address(2))
+# define CALLER_ADDR3 ((unsigned long)__builtin_return_address(3))
+# define CALLER_ADDR4 ((unsigned long)__builtin_return_address(4))
+# define CALLER_ADDR5 ((unsigned long)__builtin_return_address(5))
+#else
+# define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
+# define CALLER_ADDR1 0UL
+# define CALLER_ADDR2 0UL
+# define CALLER_ADDR3 0UL
+# define CALLER_ADDR4 0UL
+# define CALLER_ADDR5 0UL
+#endif
+
+#ifdef CONFIG_CRITICAL_IRQSOFF_TIMING
+  extern void notrace time_hardirqs_on(unsigned long a0, unsigned long a1);
+  extern void notrace time_hardirqs_off(unsigned long a0, unsigned long a1);
+#else
+# define time_hardirqs_on(a0, a1)		do { } while (0)
+# define time_hardirqs_off(a0, a1)		do { } while (0)
+#endif
+
 #endif /* _LINUX_MCOUNT_H */

Index: linux-mcount.git/kernel/fork.c
===================================================================
@@ -1036,7 +1036,7 @@ static struct task_struct *copy_process(
 
 	rt_mutex_init_task(p);
 
-#ifdef CONFIG_TRACE_IRQFLAGS
+#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_LOCKDEP)
 	DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
 	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
 #endif

Index: linux-mcount.git/kernel/lockdep.c
===================================================================
@@ -39,6 +39,7 @@
 #include <linux/irqflags.h>
 #include <linux/utsname.h>
 #include <linux/hash.h>
+#include <linux/mcount.h>
 
 #include <asm/sections.h>
 

@@ -2009,7 +2010,7 @@ void early_boot_irqs_on(void)
 /*
  * Hardirqs will be enabled:
  */
-void trace_hardirqs_on(void)
+void notrace trace_hardirqs_on_caller(unsigned long a0)
 {
 	struct task_struct *curr = current;
 	unsigned long ip;

@@ -2050,14 +2051,19 @@ void trace_hardirqs_on(void)
 	curr->hardirq_enable_ip = ip;
 	curr->hardirq_enable_event = ++curr->irq_events;
 	debug_atomic_inc(&hardirqs_on_events);
+	time_hardirqs_on(CALLER_ADDR0, a0);
 }
+EXPORT_SYMBOL(trace_hardirqs_on_caller);
 
+void notrace trace_hardirqs_on(void) {
+	trace_hardirqs_on_caller(CALLER_ADDR0);
+}
 EXPORT_SYMBOL(trace_hardirqs_on);
 
 /*
  * Hardirqs were disabled:
  */
-void trace_hardirqs_off(void)
+void notrace trace_hardirqs_off_caller(unsigned long a0)
 {
 	struct task_struct *curr = current;
 

@@ -2075,9 +2081,15 @@ void trace_hardirqs_off(void)
 		curr->hardirq_disable_ip = _RET_IP_;
 		curr->hardirq_disable_event = ++curr->irq_events;
 		debug_atomic_inc(&hardirqs_off_events);
+		time_hardirqs_off(CALLER_ADDR0, a0);
 	} else
 		debug_atomic_inc(&redundant_hardirqs_off);
 }
+EXPORT_SYMBOL(trace_hardirqs_off_caller);
+
+void notrace trace_hardirqs_off(void) {
+	trace_hardirqs_off_caller(CALLER_ADDR0);
+}
 
 EXPORT_SYMBOL(trace_hardirqs_off);
 

416 lines not on this page · raw message

87 lines not on this page · raw message

Index: linux-mcount.git/lib/tracing/tracer.h
===================================================================
@@ -112,6 +112,8 @@ extern unsigned long tracing_max_latency
 extern unsigned long tracing_thresh;
 
 void update_max_tr(struct tracing_trace *tr, struct task_struct *tsk, int cpu);
+void update_max_tr_single(struct tracing_trace *tr,
+			  struct task_struct *tsk, int cpu);
 
 static inline notrace cycle_t now(void)
 {

304 lines not on this page · raw message

Index: linux-mcount.git/include/linux/preempt.h
===================================================================
@@ -10,7 +10,7 @@
 #include <linux/linkage.h>
 #include <linux/list.h>
 
-#ifdef CONFIG_DEBUG_PREEMPT
+#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_CRITICAL_PREEMPT_TIMING)
   extern void fastcall add_preempt_count(int val);
   extern void fastcall sub_preempt_count(int val);
 #else

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -66,6 +66,7 @@
 #include <linux/unistd.h>
 #include <linux/pagemap.h>
 #include <linux/hrtimer.h>
+#include <linux/mcount.h>
 
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>

@@ -3780,26 +3781,44 @@ void scheduler_tick(void)
 #endif
 }
 
-#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)
+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
+				defined(CONFIG_CRITICAL_PREEMPT_TIMING))
+
+static inline unsigned long get_parent_ip(unsigned long addr)
+{
+	if (in_lock_functions(addr)) {
+		addr = CALLER_ADDR2;
+		if (in_lock_functions(addr))
+			addr = CALLER_ADDR3;
+	}
+	return addr;
+}
 
 void fastcall add_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
 	 */
 	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
 		return;
+#endif
 	preempt_count() += val;
+#ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Spinlock count overflowing soon?
 	 */
 	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
 				PREEMPT_MASK - 10);
+#endif
+	if (preempt_count() == val)
+		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
 EXPORT_SYMBOL(add_preempt_count);
 
 void fastcall sub_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
 	/*
 	 * Underflow?
 	 */

@@ -3811,7 +3830,10 @@ void fastcall sub_preempt_count(int val)
 	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
 			!(preempt_count() & PREEMPT_MASK)))
 		return;
+#endif
 
+	if (preempt_count() == val)
+		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 	preempt_count() -= val;
 }
 EXPORT_SYMBOL(sub_preempt_count);

Index: linux-mcount.git/arch/x86/kernel/process_32.c
===================================================================
@@ -195,7 +195,10 @@ void cpu_idle(void)
 				play_dead();
 
 			__get_cpu_var(irq_stat).idle_timestamp = jiffies;
+			/* Don't trace irqs off for idle */
+			stop_critical_timings();
 			idle();
+			start_critical_timings();
 		}
 		tick_nohz_restart_sched_tick();
 		preempt_enable_no_resched();

Index: linux-mcount.git/include/linux/mcount.h
===================================================================
@@ -58,4 +58,12 @@ extern void mcount(void);
 # define time_hardirqs_off(a0, a1)		do { } while (0)
 #endif
 
+#ifdef CONFIG_CRITICAL_PREEMPT_TIMING
+  extern void notrace trace_preempt_on(unsigned long a0, unsigned long a1);
+  extern void notrace trace_preempt_off(unsigned long a0, unsigned long a1);
+#else
+# define trace_preempt_on(a0, a1)		do { } while (0)
+# define trace_preempt_off(a0, a1)		do { } while (0)
+#endif
+
 #endif /* _LINUX_MCOUNT_H */

Index: linux-mcount.git/include/linux/irqflags.h
===================================================================
@@ -54,7 +54,8 @@
 # define INIT_TRACE_IRQFLAGS
 #endif
 
-#ifdef CONFIG_CRITICAL_IRQSOFF_TIMING
+#if defined(CONFIG_CRITICAL_IRQSOFF_TIMING) || \
+	defined(CONFIG_CRITICAL_PREEMPT_TIMING)
  extern void stop_critical_timings(void);
  extern void start_critical_timings(void);
 #else

Index: linux-mcount.git/arch/x86/kernel/apic_32.c
===================================================================
@@ -581,6 +581,8 @@ notrace fastcall void smp_apic_timer_int
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
+	trace_mark(arch_apic_timer, "ip %lx", regs->eip);
+
 	/*
 	 * NOTE! We'd better ACK the irq immediately,
 	 * because timer handling can be slow.

Index: linux-mcount.git/arch/x86/kernel/irq_32.c
===================================================================
@@ -85,6 +85,7 @@ fastcall unsigned int do_IRQ(struct pt_r
 
 	old_regs = set_irq_regs(regs);
 	irq_enter();
+	trace_mark(arch_do_irq, "ip %lx irq %d", regs->eip, irq);
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 	/* Debugging check for stack overflow: is there less than 1KB free? */
 	{

Index: linux-mcount.git/arch/x86/kernel/irq_64.c
===================================================================
@@ -149,6 +149,8 @@ asmlinkage unsigned int do_IRQ(struct pt
 	irq_enter();
 	irq = __get_cpu_var(vector_irq)[vector];
 
+	trace_mark(arch_do_irq, "ip %lx irq %d", regs->rip, irq);
+
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 	stack_overflow_check(regs);
 #endif

Index: linux-mcount.git/arch/x86/kernel/traps_32.c
===================================================================
@@ -769,6 +769,8 @@ fastcall __kprobes void do_nmi(struct pt
 
 	nmi_enter();
 
+	trace_mark(arch_do_nmi, "ip %lx flags %lx", regs->eip, regs->eflags);
+
 	cpu = smp_processor_id();
 
 	++nmi_count(cpu);

Index: linux-mcount.git/arch/x86/kernel/traps_64.c
===================================================================
@@ -782,6 +782,8 @@ asmlinkage __kprobes void default_do_nmi
 
 	cpu = smp_processor_id();
 
+	trace_mark(arch_do_nmi, "ip %lx flags %lx", regs->rip, regs->eflags);
+
 	/* Only the BSP gets external NMIs from the system.  */
 	if (!cpu)
 		reason = get_nmi_reason();

Index: linux-mcount.git/arch/x86/mm/fault_32.c
===================================================================
@@ -311,6 +311,9 @@ fastcall void __kprobes do_page_fault(st
 	/* get the address */
         address = read_cr2();
 
+	trace_mark(arch_do_page_fault, "ip %lx err %lx addr %lx",
+		   regs->eip, error_code, address);
+
 	tsk = current;
 
 	si_code = SEGV_MAPERR;

Index: linux-mcount.git/arch/x86/mm/fault_64.c
===================================================================
@@ -316,6 +316,9 @@ asmlinkage void __kprobes do_page_fault(
 	/* get the address */
 	address = read_cr2();
 
+	trace_mark(arch_do_page_fault, "ip %lx err %lx addr %lx",
+		   regs->rip, error_code, address);
+
 	info.si_code = SEGV_MAPERR;
 
 

Index: linux-mcount.git/kernel/hrtimer.c
===================================================================
@@ -709,6 +709,8 @@ static void enqueue_hrtimer(struct hrtim
 	struct hrtimer *entry;
 	int leftmost = 1;
 
+	trace_mark(kernel_hrtimer_enqueue,
+		   "expires %p timer %p", &timer->expires, timer);
 	/*
 	 * Find the right place in the rbtree:
 	 */

@@ -1130,6 +1132,7 @@ void hrtimer_interrupt(struct clock_even
 
  retry:
 	now = ktime_get();
+	trace_mark(kernel_hrtimer_interrupt, "now %p", &now);
 
 	expires_next.tv64 = KTIME_MAX;
 

@@ -1168,6 +1171,10 @@ void hrtimer_interrupt(struct clock_even
 				continue;
 			}
 
+			trace_mark(kernel_hrtimer_interrupt_expire,
+				   "expires %p timer %p",
+				   &timer->expires, timer);
+
 			__run_hrtimer(timer);
 		}
 		spin_unlock(&cpu_base->lock);

Index: linux-mcount.git/kernel/sched.c
===================================================================
@@ -90,6 +90,11 @@ unsigned long long __attribute__((weak))
 #define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
 #define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)
 
+#define __PRIO(prio) \
+	((prio) <= 99 ? 199 - (prio) : (prio) - 120)
+
+#define PRIO(p) __PRIO((p)->prio)
+
 /*
  * 'User priority' is the nice value converted to something we
  * can work with better when scaling various scheduler parameters,

@@ -1372,6 +1377,9 @@ static void activate_task(struct rq *rq,
 	if (p->state == TASK_UNINTERRUPTIBLE)
 		rq->nr_uninterruptible--;
 
+	trace_mark(kernel_sched_activate_task,
+		   "pid %d prio %d nr_running %ld",
+		   p->pid, PRIO(p), rq->nr_running);
 	enqueue_task(rq, p, wakeup);
 	inc_nr_running(p, rq);
 }

@@ -1385,6 +1393,9 @@ static void deactivate_task(struct rq *r
 		rq->nr_uninterruptible++;
 
 	dequeue_task(rq, p, sleep);
+	trace_mark(kernel_sched_deactivate_task,
+		   "pid %d prio %d nr_running %ld",
+		   p->pid, PRIO(p), rq->nr_running);
 	dec_nr_running(p, rq);
 }
 

Index: linux-mcount.git/lib/tracing/trace_events.c
===================================================================
@@ -0,0 +1,478 @@
+/*
+ * trace task events
+ *
+ * Copyright (C) 2007 Steven Rostedt <[email protected]>
+ *
+ * Based on code from the latency_tracer, that is:
+ *
+ *  Copyright (C) 2004-2006 Ingo Molnar
+ *  Copyright (C) 2004 William Lee Irwin III
+ */
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/kallsyms.h>
+#include <linux/uaccess.h>
+#include <linux/mcount.h>
+
+#include "tracer.h"
+
+static struct tracing_trace *tracer_trace __read_mostly;
+static int trace_enabled __read_mostly;
+
+static void notrace event_reset(struct tracing_trace *tr)
+{
+	struct tracing_trace_cpu *data;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		data = tr->data[cpu];
+		tracing_reset(data);
+	}
+
+	tr->time_start = now();
+}
+
+static void notrace event_trace_sched_switch(void *private,
+					     struct task_struct *prev,
+					     struct task_struct *next)
+{
+	struct tracing_trace **ptr = private;
+	struct tracing_trace *tr = *ptr;
+	struct tracing_trace_cpu *data;
+	unsigned long flags;
+	long disabled;
+	int cpu;
+
+	if (!trace_enabled || !tr)
+		return;
+
+	local_irq_save(flags);
+	cpu = raw_smp_processor_id();
+	data = tr->data[cpu];
+
+	disabled = atomic_inc_return(&data->disabled);
+	if (unlikely(disabled != 1))
+		goto out;
+
+	tracing_sched_switch_trace(tr, data, prev, next, flags);
+
+ out:
+	atomic_dec(&data->disabled);
+	local_irq_restore(flags);
+}
+
+static struct tracer_switch_ops switch_ops __read_mostly = {
+	.func = event_trace_sched_switch,
+	.private = &tracer_trace,
+};
+
+notrace int trace_event_enabled(void)
+{
+	return trace_enabled && tracer_trace;
+}
+
+/* Taken from sched.c */
+#define __PRIO(prio) \
+	((prio) <= 99 ? 199 - (prio) : (prio) - 120)
+
+#define PRIO(p) __PRIO((p)->prio)
+
+notrace void trace_event_wakeup(unsigned long ip,
+				struct task_struct *p,
+				struct task_struct *curr)
+{
+	struct tracing_trace *tr = tracer_trace;
+	struct tracing_trace_cpu *data;
+	unsigned long flags;
+	long disabled;
+	int cpu;
+
+	if (!trace_enabled || !tr)
+		return;
+
+	local_irq_save(flags);
+	cpu = raw_smp_processor_id();
+	data = tr->data[cpu];
+
+	disabled = atomic_inc_return(&data->disabled);
+	if (unlikely(disabled != 1))
+		goto out;
+
+	/* record process's command line */
+	tracing_record_cmdline(p);
+	tracing_record_cmdline(curr);
+	tracing_trace_pid(tr, data, flags, ip, p->pid, PRIO(p), PRIO(curr));
+
+ out:
+	atomic_dec(&data->disabled);
+	local_irq_restore(flags);
+}
+
+struct event_probes {
+	const char *name;
+	const char *fmt;
+	void (*func)(const struct event_probes *probe,
+		     struct tracing_trace *tr,
+		     struct tracing_trace_cpu *data,
+		     unsigned long flags,
+		     unsigned long ip,
+		     va_list ap);
+	int active;
+	int armed;
+};
+
+#define getarg(arg, ap) arg = va_arg(ap, typeof(arg))
+
+static notrace void event_trace_apic_timer(const struct event_probes *probe,
+					   struct tracing_trace *tr,
+					   struct tracing_trace_cpu *data,
+					   unsigned long flags,
+					   unsigned long ip,
+					   va_list ap)
+{
+	unsigned long parent_ip;
+
+	getarg(parent_ip, ap);
+
+	tracing_trace_special(tr, data, flags, ip, parent_ip, 0, 0);
+}
+
+static notrace void event_trace_do_irq(const struct event_probes *probe,
+				       struct tracing_trace *tr,
+				       struct tracing_trace_cpu *data,
+				       unsigned long flags,
+				       unsigned long ip,
+				       va_list ap)
+{
+	unsigned long parent_ip;
+	int irq;
+
+	getarg(parent_ip, ap);
+	getarg(irq, ap);
+
+	tracing_trace_special(tr, data, flags, ip, parent_ip, irq, 0);
+}
+
+static notrace void event_trace_do_page_fault(const struct event_probes *probe,
+					      struct tracing_trace *tr,
+					      struct tracing_trace_cpu *data,
+					      unsigned long flags,
+					      unsigned long ip,
+					      va_list ap)
+{
+	unsigned long parent_ip;
+	unsigned long err;
+	unsigned long addr;
+
+	getarg(parent_ip, ap);
+	getarg(err, ap);
+	getarg(addr, ap);
+
+	tracing_trace_special(tr, data, flags, ip, parent_ip, err, addr);
+}
+
+static notrace void event_trace_hrtimer_timer(const struct event_probes *probe,
+					      struct tracing_trace *tr,
+					      struct tracing_trace_cpu *data,
+					      unsigned long flags,
+					      unsigned long ip,
+					      va_list ap)
+{
+	ktime_t *expires;
+	unsigned long timer;
+
+	getarg(expires, ap);
+	getarg(timer, ap);
+
+	tracing_trace_timer(tr, data, flags, ip, ktime_to_ns(*expires), timer);
+}
+
+static notrace void event_trace_hrtimer_time(const struct event_probes *probe,
+					     struct tracing_trace *tr,
+					     struct tracing_trace_cpu *data,
+					     unsigned long flags,
+					     unsigned long ip,
+					     va_list ap)
+{
+	ktime_t *now;
+
+	getarg(now, ap);
+
+	tracing_trace_timer(tr, data, flags, ip, ktime_to_ns(*now), 0);
+}
+
+static notrace void event_trace_sched3(const struct event_probes *probe,
+				       struct tracing_trace *tr,
+				       struct tracing_trace_cpu *data,
+				       unsigned long flags,
+				       unsigned long ip,
+				       va_list ap)
+{
+	int pid;
+	int prio;
+	int running;
+
+	getarg(pid, ap);
+	getarg(prio, ap);
+	getarg(running, ap);
+
+	tracing_trace_pid(tr, data, flags, ip, pid, prio, running);
+}
+
+#ifndef CONFIG_WAKEUP_TRACER
+static notrace event_trace_sched_wakeup(const struct event_probes *probe,
+					struct tracing_trace *tr,
+					struct tracing_trace_cpu *data,
+					unsigned long flags,
+					unsigned long ip,
+					va_list ap)
+{
+	struct task_struct *p;
+	struct task_struct *curr;
+
+	getarg(p, ap);
+	getarg(curr, ap);
+
+	tracing_record_cmdline(p);
+	tracing_record_cmdline(curr);
+	tracing_trace_pid(tr, data, flags, ip, p->pid, PRIO(p), PRIO(curr));
+}
+#endif
+
+static struct event_probes event_probes[] __read_mostly = {
+	{
+		.name = "arch_apic_timer",
+		.fmt = "ip %lx",
+		.func = event_trace_apic_timer,
+	},
+	{
+		.name = "arch_do_irq",
+		.fmt = "ip %lx irq %d",
+		.func = event_trace_do_irq,
+	},
+	{
+		.name = "arch_do_page_fault",
+		.fmt = "ip %lx err %lx addr %lx",
+		.func = event_trace_do_page_fault,
+	},
+	{
+		.name = "kernel_hrtimer_enqueue",
+		.fmt = "expires %p timer %p",
+		.func = event_trace_hrtimer_timer,
+	},
+	{
+		.name = "kernel_hrtimer_interrupt",
+		.fmt = "now %p",
+		.func = event_trace_hrtimer_time,
+	},
+	{
+		.name = "kernel_hrtimer_interrupt_expire",
+		.fmt = "expires %p timer %p",
+		.func = event_trace_hrtimer_timer,
+	},
+	{
+		.name = "kernel_sched_activate_task",
+		.fmt = "pid %d prio %d nr_running %ld",
+		.func = event_trace_sched3,
+	},
+	{
+		.name = "kernel_sched_deactivate_task",
+		.fmt = "pid %d prio %d nr_running %ld",
+		.func = event_trace_sched3,
+	},
+#ifndef CONFIG_WAKEUP_TRACER
+	{
+		.name = "kernel_sched_wakeup",
+		.fmt = "p %p rq->curr %p",
+		.func = event_trace_sched_wakeup,
+	},
+#endif
+	{
+		.name = NULL,
+	}
+};
+
+static notrace void event_trace_callback(const struct marker *mdata,
+					 void *private_data,
+					 const char *format, ...)
+{
+	struct event_probes *probe = mdata->private;
+	struct tracing_trace *tr = tracer_trace;
+	struct tracing_trace_cpu *data;
+	unsigned long flags;
+	unsigned long ip;
+	long disabled;
+	int cpu;
+	va_list ap;
+
+	if (!trace_enabled || !tracer_trace)
+		return;
+
+	local_irq_save(flags);
+	cpu = raw_smp_processor_id();
+	data = tr->data[cpu];
+
+	disabled = atomic_inc_return(&data->disabled);
+	if (unlikely(disabled != 1))
+		goto out;
+
+	ip = CALLER_ADDR0;
+
+	va_start(ap, format);
+	probe->func(probe, tr, data, flags, ip, ap);
+	va_end(ap);
+
+ out:
+	atomic_dec(&data->disabled);
+	local_irq_restore(flags);
+}
+
+static notrace void event_trace_arm(void)
+{
+	struct event_probes *probe;
+	int ret;
+
+	for (probe = event_probes; probe->name; probe++) {
+		if (!probe->active || probe->armed)
+			continue;
+
+		ret = marker_arm(probe->name);
+		if (ret)
+			pr_info("event trace: Couldn't arm probe %s\n",
+				probe->name);
+		else
+			probe->armed = 1;
+	}
+
+	tracing_start_wakeup();
+}
+
+static notrace void event_trace_disarm(void)
+{
+	struct event_probes *probe;
+
+	tracing_stop_wakeup();
+
+	for (probe = event_probes; probe->name; probe++) {
+		if (!probe->active || !probe->armed)
+			continue;
+
+		marker_disarm(probe->name);
+		probe->armed = 0;
+	}
+}
+
+static notrace void start_event_trace(struct tracing_trace *tr)
+{
+	event_trace_arm();
+
+	event_reset(tr);
+	trace_enabled = 1;
+
+	register_tracer_switch(&switch_ops);
+	tracing_start_sched_switch();
+	tracing_start_function_trace();
+}
+
+static notrace void stop_event_trace(struct tracing_trace *tr)
+{
+	tracing_stop_function_trace();
+	trace_enabled = 0;
+	unregister_tracer_switch(&switch_ops);
+	tracing_stop_sched_switch();
+	event_trace_disarm();
+}
+
+static notrace void event_trace_init(struct tracing_trace *tr)
+{
+	tracer_trace = tr;
+
+	if (tr->ctrl)
+		start_event_trace(tr);
+}
+
+static notrace void event_trace_reset(struct tracing_trace *tr)
+{
+	if (tr->ctrl)
+		stop_event_trace(tr);
+}
+
+static void event_trace_ctrl_update(struct tracing_trace *tr)
+{
+	if (tr->ctrl)
+		start_event_trace(tr);
+	else
+		stop_event_trace(tr);
+}
+
+static void notrace event_trace_open(struct tracing_iterator *iter)
+{
+	/* stop the trace while dumping */
+	if (iter->tr->ctrl)
+		stop_event_trace(iter->tr);
+}
+
+static void notrace event_trace_close(struct tracing_iterator *iter)
+{
+	if (iter->tr->ctrl)
+		start_event_trace(iter->tr);
+}
+
+static struct trace_types_struct event_trace __read_mostly =
+{
+	.name = "events",
+	.init = event_trace_init,
+	.reset = event_trace_reset,
+	.open = event_trace_open,
+	.close = event_trace_close,
+	.ctrl_update = event_trace_ctrl_update,
+};
+
+notrace void trace_event_register(struct tracing_trace *tr)
+{
+	tracer_trace = tr;
+	event_trace_arm();
+}
+
+notrace void trace_event_unregister(struct tracing_trace *tr)
+{
+	event_trace_disarm();
+}
+
+notrace void trace_start_events(void)
+{
+	trace_enabled = 1;
+}
+
+notrace void trace_stop_events(void)
+{
+	trace_enabled = 0;
+}
+
+__init static int init_event_trace(void)
+{
+	struct event_probes *probe;
+	int ret;
+
+	ret = register_trace(&event_trace);
+	if (ret)
+		return ret;
+
+	for (probe = event_probes; probe->name; probe++) {
+
+		ret = marker_probe_register(probe->name,
+					    probe->fmt,
+					    event_trace_callback,
+					    probe);
+		if (ret)
+			pr_info("event trace: Couldn't add marker"
+				" probe to %s\n", probe->name);
+		else
+			probe->active = 1;
+	}
+
+	return 0;
+}
+
+device_initcall(init_event_trace);