Skip to contentSkip to conversation

various perf counter bits

PATCH In review Peter Zijlstra 3 participants 39 messages Apr 6, 2009 – Apr 7, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover various perf counter bits

new features:
- event limit
- record time stamps
- SIGIO support

fixes:
- mlock accounting
- swcounter task clock vs NMI readout deadlock
- comments and such

#1 lore raw

PATCH 1/15 perf_counter: update mmap() counter read, take 2

Update the userspace read method.

Paul noted that:
- userspace cannot observe ->lock & 1 on the same cpu.
- we need a barrier() between reading ->lock and ->index to ensure we read them in that prticular order.

---
 include/linux/perf_counter.h |   24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)
include/linux/perf_counter.h +10 -14
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -170,22 +170,18 @@ struct perf_counter_mmap_page {
 	 *   u32 seq;
 	 *   s64 count;
 	 *
-	 * again:
-	 *   seq = pc->lock;
-	 *   if (unlikely(seq & 1)) {
-	 *     cpu_relax();
-	 *     goto again;
-	 *   }
+	 *   do {
+	 *     seq = pc->lock;
 	 *
-	 *   if (pc->index) {
-	 *     count = pmc_read(pc->index - 1);
-	 *     count += pc->offset;
-	 *   } else
-	 *     goto regular_read;
+	 *     barrier()
+	 *     if (pc->index) {
+	 *       count = pmc_read(pc->index - 1);
+	 *       count += pc->offset;
+	 *     } else
+	 *       goto regular_read;
 	 *
-	 *   barrier();
-	 *   if (pc->lock != seq)
-	 *     goto again;
+	 *     barrier();
+	 *   } while (pc->lock != seq);
 	 *
 	 * NOTE: for obvious reason this only works on self-monitoring
 	 *       processes.
#2 lore raw

[tip:perfcounters/core] perf_counter: update mmap() counter read, take 2

Commit-ID:  a2e87d06ddbe6e6fdb8d6d2e5e985efe4efb07dd
Gitweb:     http://git.kernel.org/tip/a2e87d06ddbe6e6fdb8d6d2e5e985efe4efb07dd
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:44:59 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:54 +0200

perf_counter: update mmap() counter read, take 2

Update the userspace read method.

Paul noted that:
- userspace cannot observe ->lock & 1 on the same cpu.
- we need a barrier() between reading ->lock and ->index to ensure we read them in that prticular order.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |   24 ++++++++++--------------
 1 files changed, 10 insertions(+), 14 deletions(-)
include/linux/perf_counter.h +10 -14
index f2b914d..e22ab47 100644
@@ -170,22 +170,18 @@ struct perf_counter_mmap_page {
 	 *   u32 seq;
 	 *   s64 count;
 	 *
-	 * again:
-	 *   seq = pc->lock;
-	 *   if (unlikely(seq & 1)) {
-	 *     cpu_relax();
-	 *     goto again;
-	 *   }
+	 *   do {
+	 *     seq = pc->lock;
 	 *
-	 *   if (pc->index) {
-	 *     count = pmc_read(pc->index - 1);
-	 *     count += pc->offset;
-	 *   } else
-	 *     goto regular_read;
+	 *     barrier()
+	 *     if (pc->index) {
+	 *       count = pmc_read(pc->index - 1);
+	 *       count += pc->offset;
+	 *     } else
+	 *       goto regular_read;
 	 *
-	 *   barrier();
-	 *   if (pc->lock != seq)
-	 *     goto again;
+	 *     barrier();
+	 *   } while (pc->lock != seq);
 	 *
 	 * NOTE: for obvious reason this only works on self-monitoring
 	 *       processes.
#3 lore raw

PATCH 2/15 perf_counter: add more context information

In reply to Peter Zijlstra

Change the callchain context entries to u16, so as to gain some space.

---
 include/linux/perf_counter.h |    4 ++--
 kernel/perf_counter.c        |    6 ++----
 2 files changed, 4 insertions(+), 6 deletions(-)
include/linux/perf_counter.h +2 -2
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -507,10 +507,10 @@ extern void perf_counter_mmap(unsigned l
 extern void perf_counter_munmap(unsigned long addr, unsigned long len,
 				unsigned long pgoff, struct file *file);
 
-#define MAX_STACK_DEPTH		254
+#define MAX_STACK_DEPTH		255
 
 struct perf_callchain_entry {
-	u32	nr, hv, kernel, user;
+	u16	nr, hv, kernel, user;
 	u64	ip[MAX_STACK_DEPTH];
 };
 
kernel/perf_counter.c +2 -4
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1657,9 +1657,7 @@ void perf_counter_do_pending(void)
  * Callchain support -- arch specific
  */
 
-struct perf_callchain_entry *
-__attribute__((weak))
-perf_callchain(struct pt_regs *regs)
+__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 {
 	return NULL;
 }
@@ -1819,7 +1817,7 @@ void perf_counter_output(struct perf_cou
 		callchain = perf_callchain(regs);
 
 		if (callchain) {
-			callchain_size = (2 + callchain->nr) * sizeof(u64);
+			callchain_size = (1 + callchain->nr) * sizeof(u64);
 
 			header.type |= __PERF_EVENT_CALLCHAIN;
 			header.size += callchain_size;
#4 lore raw

[tip:perfcounters/core] perf_counter: add more context information

Commit-ID:  9c03d88e328d5f28f13191622c2ea1349c36b799
Gitweb:     http://git.kernel.org/tip/9c03d88e328d5f28f13191622c2ea1349c36b799
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:00 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:54 +0200

perf_counter: add more context information

Change the callchain context entries to u16, so as to gain some space.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |    4 ++--
 kernel/perf_counter.c        |    6 ++----
 2 files changed, 4 insertions(+), 6 deletions(-)
include/linux/perf_counter.h +2 -2
index e22ab47..f9d5cf0 100644
@@ -507,10 +507,10 @@ extern void perf_counter_mmap(unsigned long addr, unsigned long len,
 extern void perf_counter_munmap(unsigned long addr, unsigned long len,
 				unsigned long pgoff, struct file *file);
 
-#define MAX_STACK_DEPTH		254
+#define MAX_STACK_DEPTH		255
 
 struct perf_callchain_entry {
-	u32	nr, hv, kernel, user;
+	u16	nr, hv, kernel, user;
 	u64	ip[MAX_STACK_DEPTH];
 };
 
kernel/perf_counter.c +2 -4
index 2a5d4f5..727624d 100644
@@ -1657,9 +1657,7 @@ void perf_counter_do_pending(void)
  * Callchain support -- arch specific
  */
 
-struct perf_callchain_entry *
-__attribute__((weak))
-perf_callchain(struct pt_regs *regs)
+__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 {
 	return NULL;
 }
@@ -1819,7 +1817,7 @@ void perf_counter_output(struct perf_counter *counter,
 		callchain = perf_callchain(regs);
 
 		if (callchain) {
-			callchain_size = (2 + callchain->nr) * sizeof(u64);
+			callchain_size = (1 + callchain->nr) * sizeof(u64);
 
 			header.type |= __PERF_EVENT_CALLCHAIN;
 			header.size += callchain_size;
#5 lore raw

PATCH 3/15 perf_counter: SIGIO support

In reply to Peter Zijlstra

provide support for fcntl() I/O availability signals.

---
 include/linux/perf_counter.h |    2 ++
 kernel/perf_counter.c        |   20 +++++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)
include/linux/perf_counter.h +2 -0
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -238,6 +238,7 @@ enum perf_event_type {
 #include <linux/rcupdate.h>
 #include <linux/spinlock.h>
 #include <linux/hrtimer.h>
+#include <linux/fs.h>
 #include <asm/atomic.h>
 
 struct task_struct;
@@ -397,6 +398,7 @@ struct perf_counter {
 
 	/* poll related */
 	wait_queue_head_t		waitq;
+	struct fasync_struct		*fasync;
 	/* optional: for NMIs */
 	struct perf_wakeup_entry	wakeup;
 
kernel/perf_counter.c +19 -1
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1526,6 +1526,22 @@ out:
 	return ret;
 }
 
+static int perf_fasync(int fd, struct file *filp, int on)
+{
+	struct perf_counter *counter = filp->private_data;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	int retval;
+
+	mutex_lock(&inode->i_mutex);
+	retval = fasync_helper(fd, filp, on, &counter->fasync);
+	mutex_unlock(&inode->i_mutex);
+
+	if (retval < 0)
+		return retval;
+
+	return 0;
+}
+
 static const struct file_operations perf_fops = {
 	.release		= perf_release,
 	.read			= perf_read,
@@ -1533,6 +1549,7 @@ static const struct file_operations perf
 	.unlocked_ioctl		= perf_ioctl,
 	.compat_ioctl		= perf_ioctl,
 	.mmap			= perf_mmap,
+	.fasync			= perf_fasync,
 };
 
 /*
@@ -1549,7 +1566,7 @@ void perf_counter_wakeup(struct perf_cou
 	rcu_read_lock();
 	data = rcu_dereference(counter->data);
 	if (data) {
-		(void)atomic_xchg(&data->wakeup, POLL_IN);
+		atomic_set(&data->wakeup, POLL_IN);
 		/*
 		 * Ensure all data writes are issued before updating the
 		 * user-space data head information. The matching rmb()
@@ -1561,6 +1578,7 @@ void perf_counter_wakeup(struct perf_cou
 	rcu_read_unlock();
 
 	wake_up_all(&counter->waitq);
+	kill_fasync(&counter->fasync, SIGIO, POLL_IN);
 }
 
 /*
#6 lore raw

[tip:perfcounters/core] perf_counter: SIGIO support

Commit-ID:  3c446b3d3b38f991f97e9d2df0ad26a60a94dcff
Gitweb:     http://git.kernel.org/tip/3c446b3d3b38f991f97e9d2df0ad26a60a94dcff
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:01 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:55 +0200

perf_counter: SIGIO support

Provide support for fcntl() I/O availability signals.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |    2 ++
 kernel/perf_counter.c        |   20 +++++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletions(-)
include/linux/perf_counter.h +2 -0
index f9d5cf0..8d5d11b 100644
@@ -238,6 +238,7 @@ enum perf_event_type {
 #include <linux/rcupdate.h>
 #include <linux/spinlock.h>
 #include <linux/hrtimer.h>
+#include <linux/fs.h>
 #include <asm/atomic.h>
 
 struct task_struct;
@@ -398,6 +399,7 @@ struct perf_counter {
 
 	/* poll related */
 	wait_queue_head_t		waitq;
+	struct fasync_struct		*fasync;
 	/* optional: for NMIs */
 	struct perf_wakeup_entry	wakeup;
 
kernel/perf_counter.c +19 -1
index 727624d..c58cc64 100644
@@ -1526,6 +1526,22 @@ out:
 	return ret;
 }
 
+static int perf_fasync(int fd, struct file *filp, int on)
+{
+	struct perf_counter *counter = filp->private_data;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	int retval;
+
+	mutex_lock(&inode->i_mutex);
+	retval = fasync_helper(fd, filp, on, &counter->fasync);
+	mutex_unlock(&inode->i_mutex);
+
+	if (retval < 0)
+		return retval;
+
+	return 0;
+}
+
 static const struct file_operations perf_fops = {
 	.release		= perf_release,
 	.read			= perf_read,
@@ -1533,6 +1549,7 @@ static const struct file_operations perf_fops = {
 	.unlocked_ioctl		= perf_ioctl,
 	.compat_ioctl		= perf_ioctl,
 	.mmap			= perf_mmap,
+	.fasync			= perf_fasync,
 };
 
 /*
@@ -1549,7 +1566,7 @@ void perf_counter_wakeup(struct perf_counter *counter)
 	rcu_read_lock();
 	data = rcu_dereference(counter->data);
 	if (data) {
-		(void)atomic_xchg(&data->wakeup, POLL_IN);
+		atomic_set(&data->wakeup, POLL_IN);
 		/*
 		 * Ensure all data writes are issued before updating the
 		 * user-space data head information. The matching rmb()
@@ -1561,6 +1578,7 @@ void perf_counter_wakeup(struct perf_counter *counter)
 	rcu_read_unlock();
 
 	wake_up_all(&counter->waitq);
+	kill_fasync(&counter->fasync, SIGIO, POLL_IN);
 }
 
 /*
#7 lore raw

PATCH 4/15 perf_counter: generalize pending infrastructure

In reply to Peter Zijlstra

Prepare the pending infrastructure to do more than wakeups.

---
 include/linux/perf_counter.h |    7 +++--
 kernel/perf_counter.c        |   53 +++++++++++++++++++++++++------------------
 2 files changed, 36 insertions(+), 24 deletions(-)
include/linux/perf_counter.h +4 -3
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -320,8 +320,9 @@ struct perf_mmap_data {
 	void 				*data_pages[0];
 };
 
-struct perf_wakeup_entry {
-	struct perf_wakeup_entry *next;
+struct perf_pending_entry {
+	struct perf_pending_entry *next;
+	void (*func)(struct perf_pending_entry *);
 };
 
 /**
@@ -400,7 +401,7 @@ struct perf_counter {
 	wait_queue_head_t		waitq;
 	struct fasync_struct		*fasync;
 	/* optional: for NMIs */
-	struct perf_wakeup_entry	wakeup;
+	struct perf_pending_entry	pending;
 
 	void (*destroy)(struct perf_counter *);
 	struct rcu_head			rcu_head;
kernel/perf_counter.c +32 -21

106 lines not on this page · raw message

#8 lore raw

[tip:perfcounters/core] perf_counter: generalize pending infrastructure

Commit-ID:  671dec5daf3b3c43c5777be282f00120a44cf37f
Gitweb:     http://git.kernel.org/tip/671dec5daf3b3c43c5777be282f00120a44cf37f
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:02 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:55 +0200

perf_counter: generalize pending infrastructure

Prepare the pending infrastructure to do more than wakeups.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |    7 +++--
 kernel/perf_counter.c        |   53 +++++++++++++++++++++++++----------------
 2 files changed, 36 insertions(+), 24 deletions(-)
include/linux/perf_counter.h +4 -3
index 8d5d11b..977fb15 100644
@@ -321,8 +321,9 @@ struct perf_mmap_data {
 	void 				*data_pages[0];
 };
 
-struct perf_wakeup_entry {
-	struct perf_wakeup_entry *next;
+struct perf_pending_entry {
+	struct perf_pending_entry *next;
+	void (*func)(struct perf_pending_entry *);
 };
 
 /**
@@ -401,7 +402,7 @@ struct perf_counter {
 	wait_queue_head_t		waitq;
 	struct fasync_struct		*fasync;
 	/* optional: for NMIs */
-	struct perf_wakeup_entry	wakeup;
+	struct perf_pending_entry	pending;
 
 	void (*destroy)(struct perf_counter *);
 	struct rcu_head			rcu_head;
kernel/perf_counter.c +32 -21

106 lines not on this page · raw message

#9 lore raw

PATCH 5/15 perf_counter: x86: self-IPI for pending work

In reply to Peter Zijlstra

Implement set_perf_counter_pending() with a self-IPI so that it will run ASAP in a usable context.

For now use a second IRQ vector, because the primary vector pokes the apic in funny ways that seem to confuse things.

---
 arch/x86/include/asm/entry_arch.h   |    1 +
 arch/x86/include/asm/hardirq.h      |    1 +
 arch/x86/include/asm/hw_irq.h       |    1 +
 arch/x86/include/asm/irq_vectors.h  |    5 +++++
 arch/x86/include/asm/perf_counter.h |    3 ++-
 arch/x86/kernel/cpu/perf_counter.c  |   14 ++++++++++++++
 arch/x86/kernel/entry_64.S          |    2 ++
 arch/x86/kernel/irq.c               |    5 +++++
 arch/x86/kernel/irqinit_32.c        |    1 +
 arch/x86/kernel/irqinit_64.c        |    1 +
 10 files changed, 33 insertions(+), 1 deletion(-)
arch/x86/include/asm/entry_arch.h +1 -0
Index: linux-2.6/arch/x86/include/asm/entry_arch.h
===================================================================
@@ -50,6 +50,7 @@ BUILD_INTERRUPT(spurious_interrupt,SPURI
 
 #ifdef CONFIG_PERF_COUNTERS
 BUILD_INTERRUPT(perf_counter_interrupt, LOCAL_PERF_VECTOR)
+BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
 #endif
 
 #ifdef CONFIG_X86_MCE_P4THERMAL
arch/x86/include/asm/hardirq.h +1 -0
Index: linux-2.6/arch/x86/include/asm/hardirq.h
===================================================================
@@ -14,6 +14,7 @@ typedef struct {
 #endif
 	unsigned int generic_irqs;	/* arch dependent */
 	unsigned int apic_perf_irqs;
+	unsigned int apic_pending_irqs;
 #ifdef CONFIG_SMP
 	unsigned int irq_resched_count;
 	unsigned int irq_call_count;
arch/x86/include/asm/hw_irq.h +1 -0
Index: linux-2.6/arch/x86/include/asm/hw_irq.h
===================================================================
@@ -30,6 +30,7 @@ extern void apic_timer_interrupt(void);
 extern void generic_interrupt(void);
 extern void error_interrupt(void);
 extern void perf_counter_interrupt(void);
+extern void perf_pending_interrupt(void);
 
 extern void spurious_interrupt(void);
 extern void thermal_interrupt(void);
arch/x86/include/asm/irq_vectors.h +5 -0
Index: linux-2.6/arch/x86/include/asm/irq_vectors.h
===================================================================
@@ -117,6 +117,11 @@
 #define GENERIC_INTERRUPT_VECTOR	0xed
 
 /*
+ * Performance monitoring pending work vector:
+ */
+#define LOCAL_PENDING_VECTOR		0xec
+
+/*
  * First APIC vector available to drivers: (vectors 0x30-0xee) we
  * start at 0x31(0x41) to spread out vectors evenly between priority
  * levels. (0x80 is the syscall vector)
arch/x86/include/asm/perf_counter.h +2 -1
Index: linux-2.6/arch/x86/include/asm/perf_counter.h
===================================================================
@@ -84,7 +84,8 @@ union cpuid10_edx {
 #define MSR_ARCH_PERFMON_FIXED_CTR2			0x30b
 #define X86_PMC_IDX_FIXED_BUS_CYCLES			(X86_PMC_IDX_FIXED + 2)
 
-#define set_perf_counter_pending()	do { } while (0)
+extern void set_perf_counter_pending(void);
+
 #define clear_perf_counter_pending()	do { } while (0)
 #define test_perf_counter_pending()	(0)
 
arch/x86/kernel/cpu/perf_counter.c +14 -0
Index: linux-2.6/arch/x86/kernel/cpu/perf_counter.c
===================================================================
@@ -849,6 +849,20 @@ void smp_perf_counter_interrupt(struct p
 	irq_exit();
 }
 
+void smp_perf_pending_interrupt(struct pt_regs *regs)
+{
+	irq_enter();
+	ack_APIC_irq();
+	inc_irq_stat(apic_pending_irqs);
+	perf_counter_do_pending();
+	irq_exit();
+}
+
+void set_perf_counter_pending(void)
+{
+	apic->send_IPI_self(LOCAL_PENDING_VECTOR);
+}
+
 void perf_counters_lapic_init(int nmi)
 {
 	u32 apic_val;
arch/x86/kernel/entry_64.S +2 -0
Index: linux-2.6/arch/x86/kernel/entry_64.S
===================================================================
@@ -1028,6 +1028,8 @@ apicinterrupt SPURIOUS_APIC_VECTOR \
 #ifdef CONFIG_PERF_COUNTERS
 apicinterrupt LOCAL_PERF_VECTOR \
 	perf_counter_interrupt smp_perf_counter_interrupt
+apicinterrupt LOCAL_PENDING_VECTOR \
+	perf_pending_interrupt smp_perf_pending_interrupt
 #endif
 
 /*
arch/x86/kernel/irq.c +5 -0
Index: linux-2.6/arch/x86/kernel/irq.c
===================================================================
@@ -67,6 +67,10 @@ static int show_other_interrupts(struct 
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
 	seq_printf(p, "  Performance counter interrupts\n");
+	seq_printf(p, "PND: ");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs);
+	seq_printf(p, "  Performance pending work\n");
 #endif
 	if (generic_interrupt_extension) {
 		seq_printf(p, "PLT: ");
@@ -171,6 +175,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
 	sum += irq_stats(cpu)->apic_timer_irqs;
 	sum += irq_stats(cpu)->irq_spurious_count;
 	sum += irq_stats(cpu)->apic_perf_irqs;
+	sum += irq_stats(cpu)->apic_pending_irqs;
 #endif
 	if (generic_interrupt_extension)
 		sum += irq_stats(cpu)->generic_irqs;
arch/x86/kernel/irqinit_32.c +1 -0
Index: linux-2.6/arch/x86/kernel/irqinit_32.c
===================================================================
@@ -166,6 +166,7 @@ static void __init apic_intr_init(void)
 	alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
 # ifdef CONFIG_PERF_COUNTERS
 	alloc_intr_gate(LOCAL_PERF_VECTOR, perf_counter_interrupt);
+	alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 # endif
 
 # ifdef CONFIG_X86_MCE_P4THERMAL
arch/x86/kernel/irqinit_64.c +1 -0
Index: linux-2.6/arch/x86/kernel/irqinit_64.c
===================================================================
@@ -156,6 +156,7 @@ static void __init apic_intr_init(void)
 	/* Performance monitoring interrupt: */
 #ifdef CONFIG_PERF_COUNTERS
 	alloc_intr_gate(LOCAL_PERF_VECTOR, perf_counter_interrupt);
+	alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 #endif
 }
 
#10 lore raw

[tip:perfcounters/core] perf_counter: x86: self-IPI for pending work

Commit-ID:  b6276f353bf490add62dcf7db0ebd75baa3e1a37
Gitweb:     http://git.kernel.org/tip/b6276f353bf490add62dcf7db0ebd75baa3e1a37
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:03 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:56 +0200

perf_counter: x86: self-IPI for pending work

Implement set_perf_counter_pending() with a self-IPI so that it will run ASAP in a usable context.

For now use a second IRQ vector, because the primary vector pokes the apic in funny ways that seem to confuse things.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/entry_arch.h   |    1 +
 arch/x86/include/asm/hardirq.h      |    1 +
 arch/x86/include/asm/hw_irq.h       |    1 +
 arch/x86/include/asm/irq_vectors.h  |    5 +++++
 arch/x86/include/asm/perf_counter.h |    3 ++-
 arch/x86/kernel/cpu/perf_counter.c  |   14 ++++++++++++++
 arch/x86/kernel/entry_64.S          |    2 ++
 arch/x86/kernel/irq.c               |    5 +++++
 arch/x86/kernel/irqinit_32.c        |    1 +
 arch/x86/kernel/irqinit_64.c        |    1 +
 10 files changed, 33 insertions(+), 1 deletions(-)
arch/x86/include/asm/entry_arch.h +1 -0
index c2e6bed..fe24d28 100644
@@ -50,6 +50,7 @@ BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
 
 #ifdef CONFIG_PERF_COUNTERS
 BUILD_INTERRUPT(perf_counter_interrupt, LOCAL_PERF_VECTOR)
+BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
 #endif
 
 #ifdef CONFIG_X86_MCE_P4THERMAL
arch/x86/include/asm/hardirq.h +1 -0
index 2545442..f5ebe2a 100644
@@ -14,6 +14,7 @@ typedef struct {
 #endif
 	unsigned int generic_irqs;	/* arch dependent */
 	unsigned int apic_perf_irqs;
+	unsigned int apic_pending_irqs;
 #ifdef CONFIG_SMP
 	unsigned int irq_resched_count;
 	unsigned int irq_call_count;
arch/x86/include/asm/hw_irq.h +1 -0
index ae80f64..7309c0a 100644
@@ -30,6 +30,7 @@ extern void apic_timer_interrupt(void);
 extern void generic_interrupt(void);
 extern void error_interrupt(void);
 extern void perf_counter_interrupt(void);
+extern void perf_pending_interrupt(void);
 
 extern void spurious_interrupt(void);
 extern void thermal_interrupt(void);
arch/x86/include/asm/irq_vectors.h +5 -0
index 3cbd79b..545bb81 100644
@@ -117,6 +117,11 @@
 #define GENERIC_INTERRUPT_VECTOR	0xed
 
 /*
+ * Performance monitoring pending work vector:
+ */
+#define LOCAL_PENDING_VECTOR		0xec
+
+/*
  * First APIC vector available to drivers: (vectors 0x30-0xee) we
  * start at 0x31(0x41) to spread out vectors evenly between priority
  * levels. (0x80 is the syscall vector)
arch/x86/include/asm/perf_counter.h +2 -1
index e2b0e66..d08dd52 100644
@@ -84,7 +84,8 @@ union cpuid10_edx {
 #define MSR_ARCH_PERFMON_FIXED_CTR2			0x30b
 #define X86_PMC_IDX_FIXED_BUS_CYCLES			(X86_PMC_IDX_FIXED + 2)
 
-#define set_perf_counter_pending()	do { } while (0)
+extern void set_perf_counter_pending(void);
+
 #define clear_perf_counter_pending()	do { } while (0)
 #define test_perf_counter_pending()	(0)
 
arch/x86/kernel/cpu/perf_counter.c +14 -0
index c74e20d..4384158 100644
@@ -849,6 +849,20 @@ void smp_perf_counter_interrupt(struct pt_regs *regs)
 	irq_exit();
 }
 
+void smp_perf_pending_interrupt(struct pt_regs *regs)
+{
+	irq_enter();
+	ack_APIC_irq();
+	inc_irq_stat(apic_pending_irqs);
+	perf_counter_do_pending();
+	irq_exit();
+}
+
+void set_perf_counter_pending(void)
+{
+	apic->send_IPI_self(LOCAL_PENDING_VECTOR);
+}
+
 void perf_counters_lapic_init(int nmi)
 {
 	u32 apic_val;
arch/x86/kernel/entry_64.S +2 -0
index 3f129d9..1d46cba 100644
@@ -1028,6 +1028,8 @@ apicinterrupt SPURIOUS_APIC_VECTOR \
 #ifdef CONFIG_PERF_COUNTERS
 apicinterrupt LOCAL_PERF_VECTOR \
 	perf_counter_interrupt smp_perf_counter_interrupt
+apicinterrupt LOCAL_PENDING_VECTOR \
+	perf_pending_interrupt smp_perf_pending_interrupt
 #endif
 
 /*
arch/x86/kernel/irq.c +5 -0
index 9c27543..d465487 100644
@@ -67,6 +67,10 @@ static int show_other_interrupts(struct seq_file *p, int prec)
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
 	seq_printf(p, "  Performance counter interrupts\n");
+	seq_printf(p, "PND: ");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs);
+	seq_printf(p, "  Performance pending work\n");
 #endif
 	if (generic_interrupt_extension) {
 		seq_printf(p, "PLT: ");
@@ -171,6 +175,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
 	sum += irq_stats(cpu)->apic_timer_irqs;
 	sum += irq_stats(cpu)->irq_spurious_count;
 	sum += irq_stats(cpu)->apic_perf_irqs;
+	sum += irq_stats(cpu)->apic_pending_irqs;
 #endif
 	if (generic_interrupt_extension)
 		sum += irq_stats(cpu)->generic_irqs;
arch/x86/kernel/irqinit_32.c +1 -0
index 925d87c..3190a6b 100644
@@ -166,6 +166,7 @@ static void __init apic_intr_init(void)
 	alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
 # ifdef CONFIG_PERF_COUNTERS
 	alloc_intr_gate(LOCAL_PERF_VECTOR, perf_counter_interrupt);
+	alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 # endif
 
 # ifdef CONFIG_X86_MCE_P4THERMAL
arch/x86/kernel/irqinit_64.c +1 -0
index 665e2ab..53ceb26 100644
@@ -156,6 +156,7 @@ static void __init apic_intr_init(void)
 	/* Performance monitoring interrupt: */
 #ifdef CONFIG_PERF_COUNTERS
 	alloc_intr_gate(LOCAL_PERF_VECTOR, perf_counter_interrupt);
+	alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
 #endif
 }
 
#11 lore raw

PATCH 6/15 perf_counter: theres more to overflow than writing events

In reply to Peter Zijlstra

Prepare for more generic overflow handling. The new perf_counter_overflow() method will handle the generic bits of the counter overflow, and can return a !0 return value, in which case the counter should be (soft) disabled, so that it won't count until it's properly disabled.

XXX: do powerpc and swcounter

---
 arch/powerpc/kernel/perf_counter.c |    2 +-
 arch/x86/kernel/cpu/perf_counter.c |    3 ++-
 include/linux/perf_counter.h       |    4 ++--
 kernel/perf_counter.c              |   29 +++++++++++++++++++++++------
 4 files changed, 28 insertions(+), 10 deletions(-)
arch/powerpc/kernel/perf_counter.c +1 -1
Index: linux-2.6/arch/powerpc/kernel/perf_counter.c
===================================================================
@@ -732,7 +732,7 @@ static void record_and_restart(struct pe
 	 * Finally record data if requested.
 	 */
 	if (record)
-		perf_counter_output(counter, 1, regs);
+		perf_counter_overflow(counter, 1, regs);
 }
 
 /*
arch/x86/kernel/cpu/perf_counter.c +2 -1
Index: linux-2.6/arch/x86/kernel/cpu/perf_counter.c
===================================================================
@@ -800,7 +800,8 @@ again:
 			continue;
 
 		perf_save_and_restart(counter);
-		perf_counter_output(counter, nmi, regs);
+		if (perf_counter_overflow(counter, nmi, regs))
+			__pmc_generic_disable(counter, &counter->hw, bit);
 	}
 
 	hw_perf_ack_status(ack);
include/linux/perf_counter.h +2 -2
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -491,8 +491,8 @@ extern int hw_perf_group_sched_in(struct
 	       struct perf_counter_context *ctx, int cpu);
 extern void perf_counter_update_userpage(struct perf_counter *counter);
 
-extern void perf_counter_output(struct perf_counter *counter,
-				int nmi, struct pt_regs *regs);
+extern int perf_counter_overflow(struct perf_counter *counter,
+				 int nmi, struct pt_regs *regs);
 /*
  * Return 1 for a software counter, 0 for a hardware counter
  */
kernel/perf_counter.c +23 -6
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1800,8 +1800,8 @@ static void perf_output_end(struct perf_
 	rcu_read_unlock();
 }
 
-void perf_counter_output(struct perf_counter *counter,
-			 int nmi, struct pt_regs *regs)
+static void perf_counter_output(struct perf_counter *counter,
+				int nmi, struct pt_regs *regs)
 {
 	int ret;
 	u64 record_type = counter->hw_event.record_type;
@@ -2034,6 +2034,17 @@ void perf_counter_munmap(unsigned long a
 }
 
 /*
+ * Generic counter overflow handling.
+ */
+
+int perf_counter_overflow(struct perf_counter *counter,
+			  int nmi, struct pt_regs *regs)
+{
+	perf_counter_output(counter, nmi, regs);
+	return 0;
+}
+
+/*
  * Generic software counter infrastructure
  */
 
@@ -2077,6 +2088,7 @@ static void perf_swcounter_set_period(st
 
 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
 {
+	enum hrtimer_restart ret = HRTIMER_RESTART;
 	struct perf_counter *counter;
 	struct pt_regs *regs;
 
@@ -2092,12 +2104,14 @@ static enum hrtimer_restart perf_swcount
 			!counter->hw_event.exclude_user)
 		regs = task_pt_regs(current);
 
-	if (regs)
-		perf_counter_output(counter, 0, regs);
+	if (regs) {
+		if (perf_counter_overflow(counter, 0, regs))
+			ret = HRTIMER_NORESTART;
+	}
 
 	hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
 
-	return HRTIMER_RESTART;
+	return ret;
 }
 
 static void perf_swcounter_overflow(struct perf_counter *counter,
@@ -2105,7 +2119,10 @@ static void perf_swcounter_overflow(stru
 {
 	perf_swcounter_update(counter);
 	perf_swcounter_set_period(counter);
-	perf_counter_output(counter, nmi, regs);
+	if (perf_counter_overflow(counter, nmi, regs))
+		/* soft-disable the counter */
+		;
+
 }
 
 static int perf_swcounter_match(struct perf_counter *counter,
#12 lore raw

[tip:perfcounters/core] perf_counter: theres more to overflow than writing events

Commit-ID:  f6c7d5fe58b4846ee0cb4b98b6042489705eced4
Gitweb:     http://git.kernel.org/tip/f6c7d5fe58b4846ee0cb4b98b6042489705eced4
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:04 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:56 +0200

perf_counter: theres more to overflow than writing events

Prepare for more generic overflow handling. The new perf_counter_overflow() method will handle the generic bits of the counter overflow, and can return a !0 return value, in which case the counter should be (soft) disabled, so that it won't count until it's properly disabled.

XXX: do powerpc and swcounter

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 arch/powerpc/kernel/perf_counter.c |    2 +-
 arch/x86/kernel/cpu/perf_counter.c |    3 ++-
 include/linux/perf_counter.h       |    4 ++--
 kernel/perf_counter.c              |   29 +++++++++++++++++++++++------
 4 files changed, 28 insertions(+), 10 deletions(-)
arch/powerpc/kernel/perf_counter.c +1 -1
index 0a4d14f..f88c35d 100644
@@ -732,7 +732,7 @@ static void record_and_restart(struct perf_counter *counter, long val,
 	 * Finally record data if requested.
 	 */
 	if (record)
-		perf_counter_output(counter, 1, regs);
+		perf_counter_overflow(counter, 1, regs);
 }
 
 /*
arch/x86/kernel/cpu/perf_counter.c +2 -1
index 4384158..1116a41 100644
@@ -800,7 +800,8 @@ again:
 			continue;
 
 		perf_save_and_restart(counter);
-		perf_counter_output(counter, nmi, regs);
+		if (perf_counter_overflow(counter, nmi, regs))
+			__pmc_generic_disable(counter, &counter->hw, bit);
 	}
 
 	hw_perf_ack_status(ack);
include/linux/perf_counter.h +2 -2
index 977fb15..ca2d4df 100644
@@ -491,8 +491,8 @@ extern int hw_perf_group_sched_in(struct perf_counter *group_leader,
 	       struct perf_counter_context *ctx, int cpu);
 extern void perf_counter_update_userpage(struct perf_counter *counter);
 
-extern void perf_counter_output(struct perf_counter *counter,
-				int nmi, struct pt_regs *regs);
+extern int perf_counter_overflow(struct perf_counter *counter,
+				 int nmi, struct pt_regs *regs);
 /*
  * Return 1 for a software counter, 0 for a hardware counter
  */
kernel/perf_counter.c +23 -6
index 0a2ade2..195e976 100644
@@ -1800,8 +1800,8 @@ static void perf_output_end(struct perf_output_handle *handle)
 	rcu_read_unlock();
 }
 
-void perf_counter_output(struct perf_counter *counter,
-			 int nmi, struct pt_regs *regs)
+static void perf_counter_output(struct perf_counter *counter,
+				int nmi, struct pt_regs *regs)
 {
 	int ret;
 	u64 record_type = counter->hw_event.record_type;
@@ -2034,6 +2034,17 @@ void perf_counter_munmap(unsigned long addr, unsigned long len,
 }
 
 /*
+ * Generic counter overflow handling.
+ */
+
+int perf_counter_overflow(struct perf_counter *counter,
+			  int nmi, struct pt_regs *regs)
+{
+	perf_counter_output(counter, nmi, regs);
+	return 0;
+}
+
+/*
  * Generic software counter infrastructure
  */
 
@@ -2077,6 +2088,7 @@ static void perf_swcounter_set_period(struct perf_counter *counter)
 
 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
 {
+	enum hrtimer_restart ret = HRTIMER_RESTART;
 	struct perf_counter *counter;
 	struct pt_regs *regs;
 
@@ -2092,12 +2104,14 @@ static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
 			!counter->hw_event.exclude_user)
 		regs = task_pt_regs(current);
 
-	if (regs)
-		perf_counter_output(counter, 0, regs);
+	if (regs) {
+		if (perf_counter_overflow(counter, 0, regs))
+			ret = HRTIMER_NORESTART;
+	}
 
 	hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
 
-	return HRTIMER_RESTART;
+	return ret;
 }
 
 static void perf_swcounter_overflow(struct perf_counter *counter,
@@ -2105,7 +2119,10 @@ static void perf_swcounter_overflow(struct perf_counter *counter,
 {
 	perf_swcounter_update(counter);
 	perf_swcounter_set_period(counter);
-	perf_counter_output(counter, nmi, regs);
+	if (perf_counter_overflow(counter, nmi, regs))
+		/* soft-disable the counter */
+		;
+
 }
 
 static int perf_swcounter_match(struct perf_counter *counter,
#13 lore raw

PATCH 7/15 perf_counter: fix the mlock accounting

In reply to Peter Zijlstra

Reading through the code I saw I forgot the finish the mlock accounting. Do so now.

---
 kernel/perf_counter.c |   33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)
kernel/perf_counter.c +21 -12
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1461,13 +1461,14 @@ static void perf_mmap_close(struct vm_ar
 
 	if (atomic_dec_and_mutex_lock(&counter->mmap_count,
 				      &counter->mmap_mutex)) {
+		vma->vm_mm->locked_vm -= counter->data->nr_pages + 1;
 		perf_mmap_data_free(counter);
 		mutex_unlock(&counter->mmap_mutex);
 	}
 }
 
 static struct vm_operations_struct perf_mmap_vmops = {
-	.open = perf_mmap_open,
+	.open  = perf_mmap_open,
 	.close = perf_mmap_close,
 	.fault = perf_mmap_fault,
 };
@@ -1499,24 +1500,32 @@ static int perf_mmap(struct file *file, 
 	if (vma->vm_pgoff != 0)
 		return -EINVAL;
 
-	locked = vma_size >>  PAGE_SHIFT;
-	locked += vma->vm_mm->locked_vm;
+	mutex_lock(&counter->mmap_mutex);
+	if (atomic_inc_not_zero(&counter->mmap_count)) {
+		if (nr_pages != counter->data->nr_pages)
+			ret = -EINVAL;
+		goto unlock;
+	}
+
+	locked = vma->vm_mm->locked_vm;
+	locked += nr_pages + 1;
 
 	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
 	lock_limit >>= PAGE_SHIFT;
 
-	if ((locked > lock_limit) && !capable(CAP_IPC_LOCK))
-		return -EPERM;
-
-	mutex_lock(&counter->mmap_mutex);
-	if (atomic_inc_not_zero(&counter->mmap_count))
-		goto out;
+	if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
+		ret = -EPERM;
+		goto unlock;
+	}
 
 	WARN_ON(counter->data);
 	ret = perf_mmap_data_alloc(counter, nr_pages);
-	if (!ret)
-		atomic_set(&counter->mmap_count, 1);
-out:
+	if (ret)
+		goto unlock;
+
+	atomic_set(&counter->mmap_count, 1);
+	vma->vm_mm->locked_vm += nr_pages + 1;
+unlock:
 	mutex_unlock(&counter->mmap_mutex);
 
 	vma->vm_flags &= ~VM_MAYWRITE;
#14 lore raw

[tip:perfcounters/core] perf_counter: fix the mlock accounting

Commit-ID:  ebb3c4c4cb81d64cc041356915ec015e2c57092a
Gitweb:     http://git.kernel.org/tip/ebb3c4c4cb81d64cc041356915ec015e2c57092a
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:05 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:57 +0200

perf_counter: fix the mlock accounting

Reading through the code I saw I forgot the finish the mlock accounting. Do so now.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 kernel/perf_counter.c |   33 +++++++++++++++++++++------------
 1 files changed, 21 insertions(+), 12 deletions(-)
kernel/perf_counter.c +21 -12
index 195e976..c841563 100644
@@ -1461,13 +1461,14 @@ static void perf_mmap_close(struct vm_area_struct *vma)
 
 	if (atomic_dec_and_mutex_lock(&counter->mmap_count,
 				      &counter->mmap_mutex)) {
+		vma->vm_mm->locked_vm -= counter->data->nr_pages + 1;
 		perf_mmap_data_free(counter);
 		mutex_unlock(&counter->mmap_mutex);
 	}
 }
 
 static struct vm_operations_struct perf_mmap_vmops = {
-	.open = perf_mmap_open,
+	.open  = perf_mmap_open,
 	.close = perf_mmap_close,
 	.fault = perf_mmap_fault,
 };
@@ -1499,24 +1500,32 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
 	if (vma->vm_pgoff != 0)
 		return -EINVAL;
 
-	locked = vma_size >>  PAGE_SHIFT;
-	locked += vma->vm_mm->locked_vm;
+	mutex_lock(&counter->mmap_mutex);
+	if (atomic_inc_not_zero(&counter->mmap_count)) {
+		if (nr_pages != counter->data->nr_pages)
+			ret = -EINVAL;
+		goto unlock;
+	}
+
+	locked = vma->vm_mm->locked_vm;
+	locked += nr_pages + 1;
 
 	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
 	lock_limit >>= PAGE_SHIFT;
 
-	if ((locked > lock_limit) && !capable(CAP_IPC_LOCK))
-		return -EPERM;
-
-	mutex_lock(&counter->mmap_mutex);
-	if (atomic_inc_not_zero(&counter->mmap_count))
-		goto out;
+	if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
+		ret = -EPERM;
+		goto unlock;
+	}
 
 	WARN_ON(counter->data);
 	ret = perf_mmap_data_alloc(counter, nr_pages);
-	if (!ret)
-		atomic_set(&counter->mmap_count, 1);
-out:
+	if (ret)
+		goto unlock;
+
+	atomic_set(&counter->mmap_count, 1);
+	vma->vm_mm->locked_vm += nr_pages + 1;
+unlock:
 	mutex_unlock(&counter->mmap_mutex);
 
 	vma->vm_flags &= ~VM_MAYWRITE;
#15 lore raw

PATCH 8/15 perf_counter: PERF_RECORD_TIME

In reply to Peter Zijlstra

By popular request, provide means to log a timestamp along with the counter overflow event.

---
 include/linux/perf_counter.h |    2 ++
 kernel/perf_counter.c        |   14 ++++++++++++++
 2 files changed, 16 insertions(+)
include/linux/perf_counter.h +2 -0
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -102,6 +102,7 @@ enum perf_counter_record_format {
 	PERF_RECORD_TID		= 1U << 1,
 	PERF_RECORD_GROUP	= 1U << 2,
 	PERF_RECORD_CALLCHAIN	= 1U << 3,
+	PERF_RECORD_TIME	= 1U << 4,
 };
 
 /*
@@ -221,6 +222,7 @@ enum perf_event_type {
 	__PERF_EVENT_TID		= PERF_RECORD_TID,
 	__PERF_EVENT_GROUP		= PERF_RECORD_GROUP,
 	__PERF_EVENT_CALLCHAIN		= PERF_RECORD_CALLCHAIN,
+	__PERF_EVENT_TIME		= PERF_RECORD_TIME,
 };
 
 #ifdef __KERNEL__
kernel/perf_counter.c +14 -0
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1823,6 +1823,7 @@ static void perf_counter_output(struct p
 	} group_entry;
 	struct perf_callchain_entry *callchain = NULL;
 	int callchain_size = 0;
+	u64 time;
 
 	header.type = PERF_EVENT_COUNTER_OVERFLOW;
 	header.size = sizeof(header);
@@ -1859,6 +1860,16 @@ static void perf_counter_output(struct p
 		}
 	}
 
+	if (record_type & PERF_RECORD_TIME) {
+		/*
+		 * Maybe do better on x86 and provide cpu_clock_nmi()
+		 */
+		time = sched_clock();
+
+		header.type |= __PERF_EVENT_TIME;
+		header.size += sizeof(u64);
+	}
+
 	ret = perf_output_begin(&handle, counter, header.size, nmi);
 	if (ret)
 		return;
@@ -1892,6 +1903,9 @@ static void perf_counter_output(struct p
 	if (callchain)
 		perf_output_copy(&handle, callchain, callchain_size);
 
+	if (record_type & PERF_RECORD_TIME)
+		perf_output_put(&handle, time);
+
 	perf_output_end(&handle);
 }
 
#16 lore raw

[tip:perfcounters/core] perf_counter: PERF_RECORD_TIME

Commit-ID:  339f7c90b8a2f3aa2dd4267e79f797999e8a3c59
Gitweb:     http://git.kernel.org/tip/339f7c90b8a2f3aa2dd4267e79f797999e8a3c59
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:06 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:57 +0200

perf_counter: PERF_RECORD_TIME

By popular request, provide means to log a timestamp along with the counter overflow event.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |    2 ++
 kernel/perf_counter.c        |   14 ++++++++++++++
 2 files changed, 16 insertions(+), 0 deletions(-)
include/linux/perf_counter.h +2 -0
index ca2d4df..928a7fa 100644
@@ -102,6 +102,7 @@ enum perf_counter_record_format {
 	PERF_RECORD_TID		= 1U << 1,
 	PERF_RECORD_GROUP	= 1U << 2,
 	PERF_RECORD_CALLCHAIN	= 1U << 3,
+	PERF_RECORD_TIME	= 1U << 4,
 };
 
 /*
@@ -221,6 +222,7 @@ enum perf_event_type {
 	__PERF_EVENT_TID		= PERF_RECORD_TID,
 	__PERF_EVENT_GROUP		= PERF_RECORD_GROUP,
 	__PERF_EVENT_CALLCHAIN		= PERF_RECORD_CALLCHAIN,
+	__PERF_EVENT_TIME		= PERF_RECORD_TIME,
 };
 
 #ifdef __KERNEL__
kernel/perf_counter.c +14 -0
index c841563..19990d1 100644
@@ -1826,6 +1826,7 @@ static void perf_counter_output(struct perf_counter *counter,
 	} group_entry;
 	struct perf_callchain_entry *callchain = NULL;
 	int callchain_size = 0;
+	u64 time;
 
 	header.type = PERF_EVENT_COUNTER_OVERFLOW;
 	header.size = sizeof(header);
@@ -1862,6 +1863,16 @@ static void perf_counter_output(struct perf_counter *counter,
 		}
 	}
 
+	if (record_type & PERF_RECORD_TIME) {
+		/*
+		 * Maybe do better on x86 and provide cpu_clock_nmi()
+		 */
+		time = sched_clock();
+
+		header.type |= __PERF_EVENT_TIME;
+		header.size += sizeof(u64);
+	}
+
 	ret = perf_output_begin(&handle, counter, header.size, nmi);
 	if (ret)
 		return;
@@ -1895,6 +1906,9 @@ static void perf_counter_output(struct perf_counter *counter,
 	if (callchain)
 		perf_output_copy(&handle, callchain, callchain_size);
 
+	if (record_type & PERF_RECORD_TIME)
+		perf_output_put(&handle, time);
+
 	perf_output_end(&handle);
 }
 
#17 lore raw

PATCH 9/15 perf_counter: counter overflow limit

In reply to Peter Zijlstra

Provide means to auto-disable the counter after 'n' overflow events.

Create the counter with hw_event.disabled = 1, and then issue an ioctl(fd, PREF_COUNTER_IOC_REFRESH, n); to set the limit and enable the counter.

---
 include/linux/perf_counter.h |   12 +++++++---
 kernel/perf_counter.c        |   51 ++++++++++++++++++++++++++++++++++---------
 2 files changed, 50 insertions(+), 13 deletions(-)
include/linux/perf_counter.h +9 -3
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -155,8 +155,9 @@ struct perf_counter_hw_event {
 /*
  * Ioctls that can be done on a perf counter fd:
  */
-#define PERF_COUNTER_IOC_ENABLE		_IO('$', 0)
-#define PERF_COUNTER_IOC_DISABLE	_IO('$', 1)
+#define PERF_COUNTER_IOC_ENABLE		_IO ('$', 0)
+#define PERF_COUNTER_IOC_DISABLE	_IO ('$', 1)
+#define PERF_COUNTER_IOC_REFRESH	_IOW('$', 2, u32)
 
 /*
  * Structure of the page that can be mapped via mmap
@@ -403,9 +404,14 @@ struct perf_counter {
 	/* poll related */
 	wait_queue_head_t		waitq;
 	struct fasync_struct		*fasync;
-	/* optional: for NMIs */
+
+	/* delayed work for NMIs and such */
+	int				pending_wakeup;
+	int				pending_disable;
 	struct perf_pending_entry	pending;
 
+	atomic_t			event_limit;
+
 	void (*destroy)(struct perf_counter *);
 	struct rcu_head			rcu_head;
 #endif
kernel/perf_counter.c +41 -10

89 lines not on this page · raw message

#18 lore raw

[tip:perfcounters/core] perf_counter: counter overflow limit

Commit-ID:  79f146415623fe74f39af67c0f6adc208939a410
Gitweb:     http://git.kernel.org/tip/79f146415623fe74f39af67c0f6adc208939a410
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:07 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:58 +0200

perf_counter: counter overflow limit

Provide means to auto-disable the counter after 'n' overflow events.

Create the counter with hw_event.disabled = 1, and then issue an ioctl(fd, PREF_COUNTER_IOC_REFRESH, n); to set the limit and enable the counter.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |   12 +++++++--
 kernel/perf_counter.c        |   51 +++++++++++++++++++++++++++++++++--------
 2 files changed, 50 insertions(+), 13 deletions(-)
include/linux/perf_counter.h +9 -3
index 928a7fa..ef4dcbf 100644
@@ -155,8 +155,9 @@ struct perf_counter_hw_event {
 /*
  * Ioctls that can be done on a perf counter fd:
  */
-#define PERF_COUNTER_IOC_ENABLE		_IO('$', 0)
-#define PERF_COUNTER_IOC_DISABLE	_IO('$', 1)
+#define PERF_COUNTER_IOC_ENABLE		_IO ('$', 0)
+#define PERF_COUNTER_IOC_DISABLE	_IO ('$', 1)
+#define PERF_COUNTER_IOC_REFRESH	_IOW('$', 2, u32)
 
 /*
  * Structure of the page that can be mapped via mmap
@@ -403,9 +404,14 @@ struct perf_counter {
 	/* poll related */
 	wait_queue_head_t		waitq;
 	struct fasync_struct		*fasync;
-	/* optional: for NMIs */
+
+	/* delayed work for NMIs and such */
+	int				pending_wakeup;
+	int				pending_disable;
 	struct perf_pending_entry	pending;
 
+	atomic_t			event_limit;
+
 	void (*destroy)(struct perf_counter *);
 	struct rcu_head			rcu_head;
 #endif
kernel/perf_counter.c +41 -10

89 lines not on this page · raw message

#19 lore raw

PATCH 10/15 perf_counter: comment the perf_event_type stuff

In reply to Peter Zijlstra

Describe the event format.

---
 include/linux/perf_counter.h |   32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
include/linux/perf_counter.h +32 -0
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -207,6 +207,20 @@ struct perf_event_header {
 
 enum perf_event_type {
 
+	/*
+	 * The MMAP events record the PROT_EXEC mappings so that we can
+	 * correlate userspace IPs to code. They have the following structure:
+	 *
+	 * struct {
+	 * 	struct perf_event_header	header;
+	 *
+	 * 	u32				pid, tid;
+	 * 	u64				addr;
+	 * 	u64				len;
+	 * 	u64				pgoff;
+	 * 	char				filename[];
+	 * };
+	 */
 	PERF_EVENT_MMAP			= 1,
 	PERF_EVENT_MUNMAP		= 2,
 
@@ -216,6 +230,24 @@ enum perf_event_type {
 	 *
 	 * These events will have types of the form:
 	 *   PERF_EVENT_COUNTER_OVERFLOW { | __PERF_EVENT_* } *
+	 *
+	 * struct {
+	 * 	struct perf_event_header	header;
+	 *
+	 * 	{ u64			ip;	  } && __PERF_EVENT_IP
+	 * 	{ u32			pid, tid; } && __PERF_EVENT_TID
+	 *
+	 * 	{ u64			nr;
+	 * 	  { u64 event, val; } 	cnt[nr];  } && __PERF_EVENT_GROUP
+	 *
+	 * 	{ u16			nr,
+	 * 				hv,
+	 * 				kernel,
+	 * 				user;
+	 * 	  u64			ips[nr];  } && __PERF_EVENT_CALLCHAIN
+	 *
+	 * 	{ u64			time;     } && __PERF_EVENT_TIME
+	 * };
 	 */
 	PERF_EVENT_COUNTER_OVERFLOW	= 1UL << 31,
 	__PERF_EVENT_IP			= PERF_RECORD_IP,
#20 lore raw

[tip:perfcounters/core] perf_counter: comment the perf_event_type stuff

Commit-ID:  0c593b3411341e3a05a61f5527df36ab02bd11e8
Gitweb:     http://git.kernel.org/tip/0c593b3411341e3a05a61f5527df36ab02bd11e8
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:08 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:59 +0200

perf_counter: comment the perf_event_type stuff

Describe the event format.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |   32 ++++++++++++++++++++++++++++++++
 1 files changed, 32 insertions(+), 0 deletions(-)
include/linux/perf_counter.h +32 -0
index ef4dcbf..8122018 100644
@@ -208,6 +208,20 @@ struct perf_event_header {
 
 enum perf_event_type {
 
+	/*
+	 * The MMAP events record the PROT_EXEC mappings so that we can
+	 * correlate userspace IPs to code. They have the following structure:
+	 *
+	 * struct {
+	 * 	struct perf_event_header	header;
+	 *
+	 * 	u32				pid, tid;
+	 * 	u64				addr;
+	 * 	u64				len;
+	 * 	u64				pgoff;
+	 * 	char				filename[];
+	 * };
+	 */
 	PERF_EVENT_MMAP			= 1,
 	PERF_EVENT_MUNMAP		= 2,
 
@@ -217,6 +231,24 @@ enum perf_event_type {
 	 *
 	 * These events will have types of the form:
 	 *   PERF_EVENT_COUNTER_OVERFLOW { | __PERF_EVENT_* } *
+	 *
+	 * struct {
+	 * 	struct perf_event_header	header;
+	 *
+	 * 	{ u64			ip;	  } && __PERF_EVENT_IP
+	 * 	{ u32			pid, tid; } && __PERF_EVENT_TID
+	 *
+	 * 	{ u64			nr;
+	 * 	  { u64 event, val; } 	cnt[nr];  } && __PERF_EVENT_GROUP
+	 *
+	 * 	{ u16			nr,
+	 * 				hv,
+	 * 				kernel,
+	 * 				user;
+	 * 	  u64			ips[nr];  } && __PERF_EVENT_CALLCHAIN
+	 *
+	 * 	{ u64			time;     } && __PERF_EVENT_TIME
+	 * };
 	 */
 	PERF_EVENT_COUNTER_OVERFLOW	= 1UL << 31,
 	__PERF_EVENT_IP			= PERF_RECORD_IP,
#21 lore raw

PATCH 11/15 perf_counter: change event defenition

In reply to Peter Zijlstra

Currently the definition of an event is slightly ambiguous. We have wakeup events, for poll() and SIGIO, which are either generated when a record crosses a page boundary (hw_events.wakeup_events == 0), or every wakeup_events new records.

Now a record can be either an counter overflow record, or a number of different things, like the mmap PROT_EXEC region notifications.

Then there is the PERF_COUNTER_IOC_REFRESH event limit, which only considers counter overflows.

This patch changes then wakeup_events and SIGIO notification to only consider overflow events. Furthermore it changes the SIGIO notification to report SIGHUP when the event limit is reached and the counter will be disabled.

---
 include/linux/perf_counter.h |    1 +
 kernel/perf_counter.c        |   22 +++++++++++++++-------
 2 files changed, 16 insertions(+), 7 deletions(-)
include/linux/perf_counter.h +1 -0
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -439,6 +439,7 @@ struct perf_counter {
 
 	/* delayed work for NMIs and such */
 	int				pending_wakeup;
+	int				pending_kill;
 	int				pending_disable;
 	struct perf_pending_entry	pending;
 
kernel/perf_counter.c +15 -7
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -1596,7 +1596,11 @@ void perf_counter_wakeup(struct perf_cou
 	rcu_read_unlock();
 
 	wake_up_all(&counter->waitq);
-	kill_fasync(&counter->fasync, SIGIO, POLL_IN);
+
+	if (counter->pending_kill) {
+		kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
+		counter->pending_kill = 0;
+	}
 }
 
 /*
@@ -1727,6 +1731,7 @@ struct perf_output_handle {
 	unsigned int		head;
 	int			wakeup;
 	int			nmi;
+	int			overflow;
 };
 
 static inline void __perf_output_wakeup(struct perf_output_handle *handle)
@@ -1741,7 +1746,7 @@ static inline void __perf_output_wakeup(
 
 static int perf_output_begin(struct perf_output_handle *handle,
 			     struct perf_counter *counter, unsigned int size,
-			     int nmi)
+			     int nmi, int overflow)
 {
 	struct perf_mmap_data *data;
 	unsigned int offset, head;
@@ -1751,8 +1756,9 @@ static int perf_output_begin(struct perf
 	if (!data)
 		goto out;
 
-	handle->counter	= counter;
-	handle->nmi	= nmi;
+	handle->counter	 = counter;
+	handle->nmi	 = nmi;
+	handle->overflow = overflow;
 
 	if (!data->nr_pages)
 		goto fail;
@@ -1816,7 +1822,7 @@ static void perf_output_end(struct perf_
 {
 	int wakeup_events = handle->counter->hw_event.wakeup_events;
 
-	if (wakeup_events) {
+	if (handle->overflow && wakeup_events) {
 		int events = atomic_inc_return(&handle->data->events);
 		if (events >= wakeup_events) {
 			atomic_sub(wakeup_events, &handle->data->events);
@@ -1891,7 +1897,7 @@ static void perf_counter_output(struct p
 		header.size += sizeof(u64);
 	}
 
-	ret = perf_output_begin(&handle, counter, header.size, nmi);
+	ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
 	if (ret)
 		return;
 
@@ -1955,7 +1961,7 @@ static void perf_counter_mmap_output(str
 {
 	struct perf_output_handle handle;
 	int size = mmap_event->event.header.size;
-	int ret = perf_output_begin(&handle, counter, size, 0);
+	int ret = perf_output_begin(&handle, counter, size, 0, 0);
 
 	if (ret)
 		return;
@@ -2084,8 +2090,10 @@ int perf_counter_overflow(struct perf_co
 	int events = atomic_read(&counter->event_limit);
 	int ret = 0;
 
+	counter->pending_kill = POLL_IN;
 	if (events && atomic_dec_and_test(&counter->event_limit)) {
 		ret = 1;
+		counter->pending_kill = POLL_HUP;
 		if (nmi) {
 			counter->pending_disable = 1;
 			perf_pending_queue(&counter->pending,
#22 lore raw

[tip:perfcounters/core] perf_counter: change event definition

Commit-ID:  4c9e25428ff46b968a30f1dfafdba550cb6e4141
Gitweb:     http://git.kernel.org/tip/4c9e25428ff46b968a30f1dfafdba550cb6e4141
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:09 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:48:59 +0200

perf_counter: change event definition

Currently the definition of an event is slightly ambiguous. We have wakeup events, for poll() and SIGIO, which are either generated when a record crosses a page boundary (hw_events.wakeup_events == 0), or every wakeup_events new records.

Now a record can be either a counter overflow record, or a number of different things, like the mmap PROT_EXEC region notifications.

Then there is the PERF_COUNTER_IOC_REFRESH event limit, which only considers counter overflows.

This patch changes then wakeup_events and SIGIO notification to only consider overflow events. Furthermore it changes the SIGIO notification to report SIGHUP when the event limit is reached and the counter will be disabled.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |    1 +
 kernel/perf_counter.c        |   22 +++++++++++++++-------
 2 files changed, 16 insertions(+), 7 deletions(-)
include/linux/perf_counter.h +1 -0
index 8122018..0f5a400 100644
@@ -439,6 +439,7 @@ struct perf_counter {
 
 	/* delayed work for NMIs and such */
 	int				pending_wakeup;
+	int				pending_kill;
 	int				pending_disable;
 	struct perf_pending_entry	pending;
 
kernel/perf_counter.c +15 -7
index c05e103..8c8eaf0 100644
@@ -1596,7 +1596,11 @@ void perf_counter_wakeup(struct perf_counter *counter)
 	rcu_read_unlock();
 
 	wake_up_all(&counter->waitq);
-	kill_fasync(&counter->fasync, SIGIO, POLL_IN);
+
+	if (counter->pending_kill) {
+		kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
+		counter->pending_kill = 0;
+	}
 }
 
 /*
@@ -1727,6 +1731,7 @@ struct perf_output_handle {
 	unsigned int		head;
 	int			wakeup;
 	int			nmi;
+	int			overflow;
 };
 
 static inline void __perf_output_wakeup(struct perf_output_handle *handle)
@@ -1741,7 +1746,7 @@ static inline void __perf_output_wakeup(struct perf_output_handle *handle)
 
 static int perf_output_begin(struct perf_output_handle *handle,
 			     struct perf_counter *counter, unsigned int size,
-			     int nmi)
+			     int nmi, int overflow)
 {
 	struct perf_mmap_data *data;
 	unsigned int offset, head;
@@ -1751,8 +1756,9 @@ static int perf_output_begin(struct perf_output_handle *handle,
 	if (!data)
 		goto out;
 
-	handle->counter	= counter;
-	handle->nmi	= nmi;
+	handle->counter	 = counter;
+	handle->nmi	 = nmi;
+	handle->overflow = overflow;
 
 	if (!data->nr_pages)
 		goto fail;
@@ -1816,7 +1822,7 @@ static void perf_output_end(struct perf_output_handle *handle)
 {
 	int wakeup_events = handle->counter->hw_event.wakeup_events;
 
-	if (wakeup_events) {
+	if (handle->overflow && wakeup_events) {
 		int events = atomic_inc_return(&handle->data->events);
 		if (events >= wakeup_events) {
 			atomic_sub(wakeup_events, &handle->data->events);
@@ -1891,7 +1897,7 @@ static void perf_counter_output(struct perf_counter *counter,
 		header.size += sizeof(u64);
 	}
 
-	ret = perf_output_begin(&handle, counter, header.size, nmi);
+	ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
 	if (ret)
 		return;
 
@@ -1955,7 +1961,7 @@ static void perf_counter_mmap_output(struct perf_counter *counter,
 {
 	struct perf_output_handle handle;
 	int size = mmap_event->event.header.size;
-	int ret = perf_output_begin(&handle, counter, size, 0);
+	int ret = perf_output_begin(&handle, counter, size, 0, 0);
 
 	if (ret)
 		return;
@@ -2084,8 +2090,10 @@ int perf_counter_overflow(struct perf_counter *counter,
 	int events = atomic_read(&counter->event_limit);
 	int ret = 0;
 
+	counter->pending_kill = POLL_IN;
 	if (events && atomic_dec_and_test(&counter->event_limit)) {
 		ret = 1;
+		counter->pending_kill = POLL_HUP;
 		if (nmi) {
 			counter->pending_disable = 1;
 			perf_pending_queue(&counter->pending,
#23 lore raw

PATCH 12/15 perf_counter: rework context time

In reply to Peter Zijlstra

Since perf_counter_context is switched along with tasks, we can maintain the context time without using the task runtime clock.

---
 include/linux/perf_counter.h |   10 +----
 kernel/perf_counter.c        |   78 ++++++++++++++++++-------------------------
 2 files changed, 37 insertions(+), 51 deletions(-)
include/linux/perf_counter.h +3 -7
Index: linux-2.6/include/linux/perf_counter.h
===================================================================
@@ -477,14 +477,10 @@ struct perf_counter_context {
 	struct task_struct	*task;
 
 	/*
-	 * time_now is the current time in nanoseconds since an arbitrary
-	 * point in the past.  For per-task counters, this is based on the
-	 * task clock, and for per-cpu counters it is based on the cpu clock.
-	 * time_lost is an offset from the task/cpu clock, used to make it
-	 * appear that time only passes while the context is scheduled in.
+	 * Context clock, runs when context enabled.
 	 */
-	u64			time_now;
-	u64			time_lost;
+	u64			time;
+	u64			timestamp;
 #endif
 };
 
kernel/perf_counter.c +34 -44

168 lines not on this page · raw message

#24 lore raw

[tip:perfcounters/core] perf_counter: rework context time

Commit-ID:  4af4998b8aa35600f4c4a4f3c3a23baca6081d02
Gitweb:     http://git.kernel.org/tip/4af4998b8aa35600f4c4a4f3c3a23baca6081d02
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:10 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:49:00 +0200

perf_counter: rework context time

Since perf_counter_context is switched along with tasks, we can maintain the context time without using the task runtime clock.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_counter.h |   10 ++----
 kernel/perf_counter.c        |   78 ++++++++++++++++++-----------------------
 2 files changed, 37 insertions(+), 51 deletions(-)
include/linux/perf_counter.h +3 -7
index 0f5a400..7f5d353 100644
@@ -477,14 +477,10 @@ struct perf_counter_context {
 	struct task_struct	*task;
 
 	/*
-	 * time_now is the current time in nanoseconds since an arbitrary
-	 * point in the past.  For per-task counters, this is based on the
-	 * task clock, and for per-cpu counters it is based on the cpu clock.
-	 * time_lost is an offset from the task/cpu clock, used to make it
-	 * appear that time only passes while the context is scheduled in.
+	 * Context clock, runs when context enabled.
 	 */
-	u64			time_now;
-	u64			time_lost;
+	u64			time;
+	u64			timestamp;
 #endif
 };
 
kernel/perf_counter.c +34 -44

168 lines not on this page · raw message

#25 lore raw

PATCH 13/15 perf_counter: rework the task clock software counter

In reply to Peter Zijlstra

Rework the task clock software counter to use the context time instead of the task runtime clock, this removes the last such user.

---
 kernel/perf_counter.c |   42 ++++++++++++------------------------------
 1 file changed, 12 insertions(+), 30 deletions(-)
kernel/perf_counter.c +12 -30
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -974,9 +974,6 @@ int perf_counter_task_disable(void)
 	curr_rq_lock_irq_save(&flags);
 	cpu = smp_processor_id();
 
-	/* force the update of the task clock: */
-	__task_delta_exec(curr, 1);
-
 	perf_counter_task_sched_out(curr, cpu);
 
 	spin_lock(&ctx->lock);
@@ -1017,9 +1014,6 @@ int perf_counter_task_enable(void)
 	curr_rq_lock_irq_save(&flags);
 	cpu = smp_processor_id();
 
-	/* force the update of the task clock: */
-	__task_delta_exec(curr, 1);
-
 	perf_counter_task_sched_out(curr, cpu);
 
 	spin_lock(&ctx->lock);
@@ -2347,38 +2341,28 @@ static const struct hw_perf_counter_ops 
  * Software counter: task time clock
  */
 
-/*
- * Called from within the scheduler:
- */
-static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
-{
-	struct task_struct *curr = counter->task;
-	u64 delta;
-
-	delta = __task_delta_exec(curr, update);
-
-	return curr->se.sum_exec_runtime + delta;
-}
-
-static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
+static void task_clock_perf_counter_update(struct perf_counter *counter)
 {
-	u64 prev;
+	u64 prev, now;
 	s64 delta;
 
-	prev = atomic64_read(&counter->hw.prev_count);
-
-	atomic64_set(&counter->hw.prev_count, now);
+	update_context_time(counter->ctx);
+	now = counter->ctx->time;
 
+	prev = atomic64_xchg(&counter->hw.prev_count, now);
 	delta = now - prev;
-
 	atomic64_add(delta, &counter->count);
 }
 
 static int task_clock_perf_counter_enable(struct perf_counter *counter)
 {
 	struct hw_perf_counter *hwc = &counter->hw;
+	u64 now;
+
+	update_context_time(counter->ctx);
+	now = counter->ctx->time;
 
-	atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0));
+	atomic64_set(&hwc->prev_count, now);
 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	hwc->hrtimer.function = perf_swcounter_hrtimer;
 	if (hwc->irq_period) {
@@ -2393,14 +2377,12 @@ static int task_clock_perf_counter_enabl
 static void task_clock_perf_counter_disable(struct perf_counter *counter)
 {
 	hrtimer_cancel(&counter->hw.hrtimer);
-	task_clock_perf_counter_update(counter,
-			task_clock_perf_counter_val(counter, 0));
+	task_clock_perf_counter_update(counter);
 }
 
 static void task_clock_perf_counter_read(struct perf_counter *counter)
 {
-	task_clock_perf_counter_update(counter,
-			task_clock_perf_counter_val(counter, 1));
+	task_clock_perf_counter_update(counter);
 }
 
 static const struct hw_perf_counter_ops perf_ops_task_clock = {
#26 lore raw

[tip:perfcounters/core] perf_counter: rework the task clock software counter

Commit-ID:  a39d6f2556c4a19f58f538c6aa28bf8faca4fcb8
Gitweb:     http://git.kernel.org/tip/a39d6f2556c4a19f58f538c6aa28bf8faca4fcb8
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:11 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:49:00 +0200

perf_counter: rework the task clock software counter

Rework the task clock software counter to use the context time instead of the task runtime clock, this removes the last such user.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 kernel/perf_counter.c |   42 ++++++++++++------------------------------
 1 files changed, 12 insertions(+), 30 deletions(-)
kernel/perf_counter.c +12 -30
index 84d85ab..56b7eb5 100644
@@ -974,9 +974,6 @@ int perf_counter_task_disable(void)
 	curr_rq_lock_irq_save(&flags);
 	cpu = smp_processor_id();
 
-	/* force the update of the task clock: */
-	__task_delta_exec(curr, 1);
-
 	perf_counter_task_sched_out(curr, cpu);
 
 	spin_lock(&ctx->lock);
@@ -1017,9 +1014,6 @@ int perf_counter_task_enable(void)
 	curr_rq_lock_irq_save(&flags);
 	cpu = smp_processor_id();
 
-	/* force the update of the task clock: */
-	__task_delta_exec(curr, 1);
-
 	perf_counter_task_sched_out(curr, cpu);
 
 	spin_lock(&ctx->lock);
@@ -2347,38 +2341,28 @@ static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
  * Software counter: task time clock
  */
 
-/*
- * Called from within the scheduler:
- */
-static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
-{
-	struct task_struct *curr = counter->task;
-	u64 delta;
-
-	delta = __task_delta_exec(curr, update);
-
-	return curr->se.sum_exec_runtime + delta;
-}
-
-static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
+static void task_clock_perf_counter_update(struct perf_counter *counter)
 {
-	u64 prev;
+	u64 prev, now;
 	s64 delta;
 
-	prev = atomic64_read(&counter->hw.prev_count);
-
-	atomic64_set(&counter->hw.prev_count, now);
+	update_context_time(counter->ctx);
+	now = counter->ctx->time;
 
+	prev = atomic64_xchg(&counter->hw.prev_count, now);
 	delta = now - prev;
-
 	atomic64_add(delta, &counter->count);
 }
 
 static int task_clock_perf_counter_enable(struct perf_counter *counter)
 {
 	struct hw_perf_counter *hwc = &counter->hw;
+	u64 now;
+
+	update_context_time(counter->ctx);
+	now = counter->ctx->time;
 
-	atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0));
+	atomic64_set(&hwc->prev_count, now);
 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	hwc->hrtimer.function = perf_swcounter_hrtimer;
 	if (hwc->irq_period) {
@@ -2393,14 +2377,12 @@ static int task_clock_perf_counter_enable(struct perf_counter *counter)
 static void task_clock_perf_counter_disable(struct perf_counter *counter)
 {
 	hrtimer_cancel(&counter->hw.hrtimer);
-	task_clock_perf_counter_update(counter,
-			task_clock_perf_counter_val(counter, 0));
+	task_clock_perf_counter_update(counter);
 }
 
 static void task_clock_perf_counter_read(struct perf_counter *counter)
 {
-	task_clock_perf_counter_update(counter,
-			task_clock_perf_counter_val(counter, 1));
+	task_clock_perf_counter_update(counter);
 }
 
 static const struct hw_perf_counter_ops perf_ops_task_clock = {
#27 lore raw

[tip:perfcounters/core] x86, perfcounters: add atomic64_xchg()

In reply to Peter Zijlstra
Commit-ID:  b226fa12991d6ee9b6b13a7e1de88ec5d25f5007
Gitweb:     http://git.kernel.org/tip/b226fa12991d6ee9b6b13a7e1de88ec5d25f5007
Author:     Ingo Molnar <[email protected]>
AuthorDate: Tue, 7 Apr 2009 11:30:17 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 11:32:43 +0200

x86, perfcounters: add atomic64_xchg()

Complete atomic64_t support on the 32-bit side by adding atomic64_xch().

Cc: Peter Zijlstra <[email protected]>
LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/atomic_32.h |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
arch/x86/include/asm/atomic_32.h +6 -0
index 977250e..a9fef6c 100644
@@ -306,6 +306,12 @@ static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
 	} while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
 }
 
+static inline void
+atomic64_xchg(atomic64_t *ptr, unsigned long long new_val)
+{
+	atomic64_set(ptr, new_val);
+}
+
 /**
  * atomic64_read - read atomic64 variable
  * @ptr:      pointer to type atomic64_t
#28 lore raw

Re: [tip:perfcounters/core] x86, perfcounters: add atomic64_xchg()

Quoted 16 lines · Ingo Molnar

Complete atomic64_t support on the 32-bit side by adding atomic64_xch().

Cc: Peter Zijlstra <[email protected]>
LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/atomic_32.h |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
arch/x86/include/asm/atomic_32.h +0 -0
index 977250e..a9fef6c 100644

arch/x86/include/asm/atomic_32.h static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val) · line 313

Quoted 8 lines
arch/x86/include/asm/atomic_32.h +5 -0
@@ -306,6 +306,12 @@ static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
 	} while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
 }
 
+static inline void
+atomic64_xchg(atomic64_t *ptr, unsigned long long new_val)
+{
+	atomic64_set(ptr, new_val);
+}

Umm, I don't know much about x86, but that doesn't look like an exchange operation to me... Shouldn't it return a value, for a start?

Paul.

#29 lore raw

Re: [tip:perfcounters/core] x86, perfcounters: add atomic64_xchg()

Quoted 21 lines · Paul Mackerras
Quoted 19 lines · Ingo Molnar

Complete atomic64_t support on the 32-bit side by adding atomic64_xch().

Cc: Peter Zijlstra <[email protected]>
LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/atomic_32.h |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
arch/x86/include/asm/atomic_32.h +0 -0
index 977250e..a9fef6c 100644
@@ -306,6 +306,12 @@ static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
 	} while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
 }
 
Quoted 9 lines

arch/x86/include/asm/atomic_32.h static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val) · line 313

Quoted 5 lines
arch/x86/include/asm/atomic_32.h +5 -0
+static inline void
+atomic64_xchg(atomic64_t *ptr, unsigned long long new_val)
+{
+	atomic64_set(ptr, new_val);
+}

Umm, I don't know much about x86, but that doesn't look like an exchange operation to me... Shouldn't it return a value, for a start?

Yes, indeed :) Fixed it.

/me officially loves commit notifications

Ingo

#30 lore raw

[tip:perfcounters/core] x86, perfcounters: add atomic64_xchg()

In reply to Peter Zijlstra
Commit-ID:  98c2aaf8be5baf7193be37fb28bce8e7327158bc
Gitweb:     http://git.kernel.org/tip/98c2aaf8be5baf7193be37fb28bce8e7327158bc
Author:     Ingo Molnar <[email protected]>
AuthorDate: Tue, 7 Apr 2009 11:30:17 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 12:02:41 +0200

x86, perfcounters: add atomic64_xchg()

Complete atomic64_t support on the 32-bit side by adding atomic64_xch().

Cc: Peter Zijlstra <[email protected]>
LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/atomic_32.h |   24 +++++++++++++++++++++---
 1 files changed, 21 insertions(+), 3 deletions(-)
arch/x86/include/asm/atomic_32.h +21 -3
index 977250e..aff9f1f 100644
@@ -291,19 +291,37 @@ atomic64_cmpxchg(atomic64_t *ptr, unsigned long long old_val,
 }
 
 /**
- * atomic64_set - set atomic64 variable
+ * atomic64_xchg - xchg atomic64 variable
  * @ptr:      pointer to type atomic64_t
  * @new_val:  value to assign
+ * @old_val:  old value that was there
  *
- * Atomically sets the value of @ptr to @new_val.
+ * Atomically xchgs the value of @ptr to @new_val and returns
+ * the old value.
  */
-static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
+
+static inline unsigned long long
+atomic64_xchg(atomic64_t *ptr, unsigned long long new_val)
 {
 	unsigned long long old_val;
 
 	do {
 		old_val = atomic_read(ptr);
 	} while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
+
+	return old_val;
+}
+
+/**
+ * atomic64_set - set atomic64 variable
+ * @ptr:      pointer to type atomic64_t
+ * @new_val:  value to assign
+ *
+ * Atomically sets the value of @ptr to @new_val.
+ */
+static inline void atomic64_set(atomic64_t *ptr, unsigned long long new_val)
+{
+	atomic64_xchg(ptr, new_val);
 }
 
 /**
#31 lore raw

PATCH 14/15 perf_counter: remove rq->lock usage

In reply to Peter Zijlstra

Now that all the task runtime clock users are gone, remove the ugly rq->lock usage from perf counters, which solves the nasty deadlock seen when a software task clock counter was read from an NMI overflow context.

---
 include/linux/kernel_stat.h |    2 --
 kernel/perf_counter.c       |   42 ++++++++++++++++--------------------------
 kernel/sched.c              |   20 --------------------
 3 files changed, 16 insertions(+), 48 deletions(-)
include/linux/kernel_stat.h +0 -2
Index: linux-2.6/include/linux/kernel_stat.h
===================================================================
@@ -85,8 +85,6 @@ static inline unsigned int kstat_irqs(un
 /*
  * Lock/unlock the current runqueue - to extract task statistics:
  */
-extern void curr_rq_lock_irq_save(unsigned long *flags);
-extern void curr_rq_unlock_irq_restore(unsigned long *flags);
 extern unsigned long long __task_delta_exec(struct task_struct *tsk, int update);
 extern unsigned long long task_delta_exec(struct task_struct *);
 
kernel/perf_counter.c +16 -26

136 lines not on this page · raw message

kernel/sched.c +0 -20
Index: linux-2.6/kernel/sched.c
===================================================================
@@ -1004,26 +1004,6 @@ static struct rq *task_rq_lock(struct ta
 	}
 }
 
-void curr_rq_lock_irq_save(unsigned long *flags)
-	__acquires(rq->lock)
-{
-	struct rq *rq;
-
-	local_irq_save(*flags);
-	rq = cpu_rq(smp_processor_id());
-	spin_lock(&rq->lock);
-}
-
-void curr_rq_unlock_irq_restore(unsigned long *flags)
-	__releases(rq->lock)
-{
-	struct rq *rq;
-
-	rq = cpu_rq(smp_processor_id());
-	spin_unlock(&rq->lock);
-	local_irq_restore(*flags);
-}
-
 void task_rq_unlock_wait(struct task_struct *p)
 {
 	struct rq *rq = task_rq(p);
#32 lore raw

[tip:perfcounters/core] perf_counter: remove rq->lock usage

Commit-ID:  849691a6cd40270ff5f4a8846d5f6bf8df663ffc
Gitweb:     http://git.kernel.org/tip/849691a6cd40270ff5f4a8846d5f6bf8df663ffc
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:12 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:49:01 +0200

perf_counter: remove rq->lock usage

Now that all the task runtime clock users are gone, remove the ugly rq->lock usage from perf counters, which solves the nasty deadlock seen when a software task clock counter was read from an NMI overflow context.

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/kernel_stat.h |    2 --
 kernel/perf_counter.c       |   42 ++++++++++++++++--------------------------
 kernel/sched.c              |   20 --------------------
 3 files changed, 16 insertions(+), 48 deletions(-)
include/linux/kernel_stat.h +0 -2
index b6d2887..080d1fd 100644
@@ -85,8 +85,6 @@ static inline unsigned int kstat_irqs(unsigned int irq)
 /*
  * Lock/unlock the current runqueue - to extract task statistics:
  */
-extern void curr_rq_lock_irq_save(unsigned long *flags);
-extern void curr_rq_unlock_irq_restore(unsigned long *flags);
 extern unsigned long long __task_delta_exec(struct task_struct *tsk, int update);
 extern unsigned long long task_delta_exec(struct task_struct *);
 
kernel/perf_counter.c +16 -26

136 lines not on this page · raw message

kernel/sched.c +0 -20
index f76e3c0..0de2f81 100644
@@ -997,26 +997,6 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
 	}
 }
 
-void curr_rq_lock_irq_save(unsigned long *flags)
-	__acquires(rq->lock)
-{
-	struct rq *rq;
-
-	local_irq_save(*flags);
-	rq = cpu_rq(smp_processor_id());
-	spin_lock(&rq->lock);
-}
-
-void curr_rq_unlock_irq_restore(unsigned long *flags)
-	__releases(rq->lock)
-{
-	struct rq *rq;
-
-	rq = cpu_rq(smp_processor_id());
-	spin_unlock(&rq->lock);
-	local_irq_restore(*flags);
-}
-
 void task_rq_unlock_wait(struct task_struct *p)
 {
 	struct rq *rq = task_rq(p);
#33 lore raw

PATCH 15/15 perf_counter: minimize context time updates

In reply to Peter Zijlstra

Push the update_context_time() calls up the stack so that we get less invokations and thereby a less noisy output:

before:

 # ./perfstat -e 1:0 -e 1:1 -e 1:1 -e 1:1 -l ls > /dev/null

Performance counter stats for 'ls':

      10.163691  cpu clock ticks      (msecs)  (scaled from 98.94%)
      10.215360  task clock ticks     (msecs)  (scaled from 98.18%)
      10.185549  task clock ticks     (msecs)  (scaled from 98.53%)
      10.183581  task clock ticks     (msecs)  (scaled from 98.71%)

Wall-clock time elapsed: 11.912858 msecs

after:

 # ./perfstat -e 1:0 -e 1:1 -e 1:1 -e 1:1 -l ls > /dev/null

Performance counter stats for 'ls':

       9.316630  cpu clock ticks      (msecs)
       9.280789  task clock ticks     (msecs)
       9.280789  task clock ticks     (msecs)
       9.280789  task clock ticks     (msecs)

Wall-clock time elapsed: 9.574872 msecs

---
 kernel/perf_counter.c |    7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/perf_counter.c +5 -2
Index: linux-2.6/kernel/perf_counter.c
===================================================================
@@ -319,6 +319,8 @@ static void __perf_counter_disable(void 
 
 	spin_lock_irqsave(&ctx->lock, flags);
 
+	update_context_time(ctx);
+
 	/*
 	 * If the counter is on, turn it off.
 	 * If it is in error state, leave it in error state.
@@ -797,6 +799,8 @@ void perf_counter_task_sched_out(struct 
 	if (likely(!cpuctx->task_ctx))
 		return;
 
+	update_context_time(ctx);
+
 	regs = task_pt_regs(task);
 	perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs);
 	__perf_counter_sched_out(ctx, cpuctx);
@@ -2336,7 +2340,6 @@ static void task_clock_perf_counter_upda
 	u64 prev, now;
 	s64 delta;
 
-	update_context_time(counter->ctx);
 	now = counter->ctx->time;
 
 	prev = atomic64_xchg(&counter->hw.prev_count, now);
@@ -2349,7 +2352,6 @@ static int task_clock_perf_counter_enabl
 	struct hw_perf_counter *hwc = &counter->hw;
 	u64 now;
 
-	update_context_time(counter->ctx);
 	now = counter->ctx->time;
 
 	atomic64_set(&hwc->prev_count, now);
@@ -2372,6 +2374,7 @@ static void task_clock_perf_counter_disa
 
 static void task_clock_perf_counter_read(struct perf_counter *counter)
 {
+	update_context_time(counter->ctx);
 	task_clock_perf_counter_update(counter);
 }
 
#34 lore raw

[tip:perfcounters/core] perf_counter: minimize context time updates

Commit-ID:  bce379bf358738ab8efc8cda2594a299ac685085
Gitweb:     http://git.kernel.org/tip/bce379bf358738ab8efc8cda2594a299ac685085
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 6 Apr 2009 11:45:13 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Tue, 7 Apr 2009 10:49:01 +0200

perf_counter: minimize context time updates

Push the update_context_time() calls up the stack so that we get less invokations and thereby a less noisy output:

before:

 # ./perfstat -e 1:0 -e 1:1 -e 1:1 -e 1:1 -l ls > /dev/null

Performance counter stats for 'ls':

      10.163691  cpu clock ticks      (msecs)  (scaled from 98.94%)
      10.215360  task clock ticks     (msecs)  (scaled from 98.18%)
      10.185549  task clock ticks     (msecs)  (scaled from 98.53%)
      10.183581  task clock ticks     (msecs)  (scaled from 98.71%)

Wall-clock time elapsed: 11.912858 msecs

after:

 # ./perfstat -e 1:0 -e 1:1 -e 1:1 -e 1:1 -l ls > /dev/null

Performance counter stats for 'ls':

       9.316630  cpu clock ticks      (msecs)
       9.280789  task clock ticks     (msecs)
       9.280789  task clock ticks     (msecs)
       9.280789  task clock ticks     (msecs)

Wall-clock time elapsed: 9.574872 msecs

Cc Paul Mackerras <[email protected]>
Cc Corey Ashford <[email protected]>

LKML-Reference: <[email protected]>

---
 kernel/perf_counter.c |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)
kernel/perf_counter.c +5 -2
index f4f7596..863703b 100644
@@ -319,6 +319,8 @@ static void __perf_counter_disable(void *info)
 
 	spin_lock_irqsave(&ctx->lock, flags);
 
+	update_context_time(ctx);
+
 	/*
 	 * If the counter is on, turn it off.
 	 * If it is in error state, leave it in error state.
@@ -797,6 +799,8 @@ void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 	if (likely(!cpuctx->task_ctx))
 		return;
 
+	update_context_time(ctx);
+
 	regs = task_pt_regs(task);
 	perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs);
 	__perf_counter_sched_out(ctx, cpuctx);
@@ -2336,7 +2340,6 @@ static void task_clock_perf_counter_update(struct perf_counter *counter)
 	u64 prev, now;
 	s64 delta;
 
-	update_context_time(counter->ctx);
 	now = counter->ctx->time;
 
 	prev = atomic64_xchg(&counter->hw.prev_count, now);
@@ -2349,7 +2352,6 @@ static int task_clock_perf_counter_enable(struct perf_counter *counter)
 	struct hw_perf_counter *hwc = &counter->hw;
 	u64 now;
 
-	update_context_time(counter->ctx);
 	now = counter->ctx->time;
 
 	atomic64_set(&hwc->prev_count, now);
@@ -2372,6 +2374,7 @@ static void task_clock_perf_counter_disable(struct perf_counter *counter)
 
 static void task_clock_perf_counter_read(struct perf_counter *counter)
 {
+	update_context_time(counter->ctx);
 	task_clock_perf_counter_update(counter);
 }
 
#35 lore raw
In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra

new features:
- event limit
- record time stamps
- SIGIO support

fixes:
- mlock accounting
- swcounter task clock vs NMI readout deadlock
- comments and such

Looks good so i've applied it, but got this build-failure on 32-bit:

kernel/perf_counter.c:2345: error: implicit declaration of function atomic64_xchg

atomic64_xchg() is not implemented yet on the 32-bit side.

Ingo

#36 lore raw

we also still have this build warning when building kerneltop:

kerneltop.c: In function ‘mmap_read’:
kerneltop.c:1263: warning: comparison of distinct pointer types lacks a cast

Ingo

#37 lore raw
In reply to Ingo Molnar
Quoted 5 lines · Ingo Molnar

Looks good so i've applied it, but got this build-failure on 32-bit:

kernel/perf_counter.c:2345: error: implicit declaration of function atomic64_xchg

atomic64_xchg() is not implemented yet on the 32-bit side.

ok, implemented it.

Ingo

#38 lore raw
In reply to Peter Zijlstra

another thing i noticed is that our CPU migration counters look a bit weird:

aldebaran:~/linux/linux/Documentation/perf_counter> taskset 1 perfstat ls design.txt kerneltop kerneltop.c Makefile perfstat

Performance counter stats for 'ls':

       0.686802  task clock ticks     (msecs)
              3  context switches     (events)
              4  CPU migrations       (events)
            285  pagefaults           (events)
        2183899  CPU cycles           (events)
        1492516  instructions         (events)
          24638  cache references     (events)
           1763  cache misses         (events)

Wall-clock time elapsed: 0.694796 msecs

the 'taskset 1' binds perfstat to a single CPU - still we had 4 cross-CPU migrations?

Ingo