Skip to contentSkip to conversation

perf and jump_label bits

RFC PATCH In review Peter Zijlstra 7 participants 32 messages Oct 14, 2010 – Oct 18, 2010

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

cover perf and jump_label bits

Only compile tested, but should fix the pending hw_breakpoint issue and use jump_labels to optimize some of perf

#7 lore raw

RFC PATCH 3/7 perf, hw_breakpoint: Fix crash in hw_breakpoint creation

hw_breakpoint creation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.

With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.

This makes hw_breakpoints unhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in the event::hw bits for now...

Cc Frederic Weisbecker <[email protected]>
---
 include/linux/perf_event.h |    7 +++++++
 kernel/hw_breakpoint.c     |    8 ++++----
 kernel/perf_event.c        |   21 ++++++++++++++++-----
 3 files changed, 27 insertions(+), 9 deletions(-)
include/linux/perf_event.h +7 -0
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -536,6 +536,12 @@ struct hw_perf_event {
 		struct { /* breakpoint */
 			struct arch_hw_breakpoint	info;
 			struct list_head		bp_list;
+			/*
+			 * Crufty hack to avoid the chicken and egg
+			 * problem hw_breakpoint has with context
+			 * creation and event initalization.
+			 */
+			struct task_struct		*bp_target;
 		};
 #endif
 	};
@@ -693,6 +699,7 @@ struct swevent_hlist {
 
 #define PERF_ATTACH_CONTEXT	0x01
 #define PERF_ATTACH_GROUP	0x02
+#define PERF_ATTACH_TASK	0x04
 
 /**
  * struct perf_event - performance event kernel representation:
kernel/hw_breakpoint.c +4 -4
Index: linux-2.6/kernel/hw_breakpoint.c
===================================================================
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(i
  */
 static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
 {
-	struct perf_event_context *ctx = bp->ctx;
+	struct task_struct *tsk = bp->hw.bp_target;
 	struct perf_event *iter;
 	int count = 0;
 
 	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-		if (iter->ctx == ctx && find_slot_idx(iter) == type)
+		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
 			count += hw_breakpoint_weight(iter);
 	}
 
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots
 		    enum bp_type_idx type)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	if (cpu >= 0) {
 		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bo
 	       int weight)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	/* Pinned counter cpu profiling */
 	if (!tsk) {
kernel/perf_event.c +16 -5
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -5189,9 +5189,10 @@ struct pmu *perf_init_event(struct perf_
  */
 static struct perf_event *
 perf_event_alloc(struct perf_event_attr *attr, int cpu,
-		   struct perf_event *group_leader,
-		   struct perf_event *parent_event,
-		   perf_overflow_handler_t overflow_handler)
+		 struct task_struct *task,
+		 struct perf_event *group_leader,
+		 struct perf_event *parent_event,
+		 perf_overflow_handler_t overflow_handler)
 {
 	struct pmu *pmu;
 	struct perf_event *event;
@@ -5233,6 +5234,15 @@ perf_event_alloc(struct perf_event_attr 
 
 	event->state		= PERF_EVENT_STATE_INACTIVE;
 
+	if (task) {
+		event->attach_state = PERF_ATTACH_TASK;
+		/*
+		 * hw_breakpoint is a bit difficult here..
+		 */
+		if (attr->type == PERF_TYPE_BREAKPOINT)
+			event->hw.bp_target = task;
+	}
+
 	if (!overflow_handler && parent_event)
 		overflow_handler = parent_event->overflow_handler;
 	
@@ -5493,7 +5503,7 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
+	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err_task;
@@ -5662,7 +5672,7 @@ perf_event_create_kernel_counter(struct 
 	 * Get the target context (task or percpu):
 	 */
 
-	event = perf_event_alloc(attr, cpu, NULL, NULL, overflow_handler);
+	event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err;
@@ -5930,6 +5940,7 @@ inherit_event(struct perf_event *parent_
 
 	child_event = perf_event_alloc(&parent_event->attr,
 					   parent_event->cpu,
+					   child,
 					   group_leader, parent_event,
 					   NULL);
 	if (IS_ERR(child_event))
#5 lore raw

RFC PATCH 2/7 perf: Find task before event alloc

In reply to Peter Zijlstra

So that we can pass the task pointer to the event allocation, so that we can use task associated data during event initialization.

---
 kernel/perf_event.c |   23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)
kernel/perf_event.c +12 -11
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -5485,10 +5485,18 @@ SYSCALL_DEFINE5(perf_event_open,
 			group_leader = NULL;
 	}
 
+	if (pid != -1) {
+		task = find_lively_task_by_vpid(pid);
+		if (IS_ERR(task)) {
+			err = PTR_ERR(task);
+			goto err_group_fd;
+		}
+	}
+
 	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
-		goto err_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5520,21 +5528,13 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	if (pid != -1) {
-		task = find_lively_task_by_vpid(pid);
-		if (IS_ERR(task)) {
-			err = PTR_ERR(task);
-			goto err_group_fd;
-		}
-	}
-
 	/*
 	 * Get the target context (task or percpu):
 	 */
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_task;
+		goto err_alloc;
 	}
 
 	/*
@@ -5630,12 +5630,13 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_alloc:
+	free_event(event);
 err_task:
 	if (task)
 		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
-	free_event(event);
 err_fd:
 	put_unused_fd(event_fd);
 	return err;
#21 lore raw

RFC PATCH 7/7 perf: Optimize sw events

In reply to Peter Zijlstra
---
 include/linux/perf_event.h |   16 +++++++++-------
 kernel/perf_event.c        |    4 ++--
 2 files changed, 11 insertions(+), 9 deletions(-)
include/linux/perf_event.h +9 -7
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 
 extern void perf_event_mmap(struct vm_area_struct *vma);
kernel/perf_event.c +2 -2
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -4603,7 +4603,7 @@ static void sw_perf_event_destroy(struct
 
 	WARN_ON(event->parent);
 
-	atomic_dec(&perf_swevent_enabled[event_id]);
+	jump_label_dec(&perf_swevent_enabled[event_id]);
 	swevent_hlist_put(event);
 }
 
@@ -4633,7 +4633,7 @@ static int perf_swevent_init(struct perf
 		if (err)
 			return err;
 
-		atomic_inc(&perf_swevent_enabled[event_id]);
+		jump_label_inc(&perf_swevent_enabled[event_id]);
 		event->destroy = sw_perf_event_destroy;
 	}
 
#17 lore raw

RFC PATCH 6/7 perf: use jump_label to optimize the scheduler hooks

In reply to Peter Zijlstra
---
 include/linux/perf_event.h |   27 +++++++++++++++++++++++++--
 kernel/perf_event.c        |   24 +++++++++---------------
 2 files changed, 34 insertions(+), 17 deletions(-)
include/linux/perf_event.h +25 -2
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/irq_work.h>
+#include <linux/jump_label_ref.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
 
@@ -893,8 +894,30 @@ struct perf_output_handle {
 extern int perf_pmu_register(struct pmu *pmu);
 extern void perf_pmu_unregister(struct pmu *pmu);
 
-extern void perf_event_task_sched_in(struct task_struct *task);
-extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+extern void __perf_event_task_sched_in(struct task_struct *task);
+extern void __perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+
+extern atomic_t perf_task_events;
+
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_in(task);
+}
+
+static inline
+void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_out(task, next);
+}
+
 extern int perf_event_init_task(struct task_struct *child);
 extern void perf_event_exit_task(struct task_struct *child);
 extern void perf_event_free_task(struct task_struct *task);
kernel/perf_event.c +9 -15
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -34,7 +34,7 @@
 
 #include <asm/irq_regs.h>
 
-static atomic_t nr_events __read_mostly;
+atomic_t perf_task_events __read_mostly;
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
@@ -1251,8 +1251,8 @@ void perf_event_context_sched_out(struct
  * accessing the event control register. If a NMI hits, then it will
  * not restart the event.
  */
-void perf_event_task_sched_out(struct task_struct *task,
-			       struct task_struct *next)
+void __perf_event_task_sched_out(struct task_struct *task,
+				 struct task_struct *next)
 {
 	int ctxn;
 
@@ -1280,14 +1280,6 @@ static void task_ctx_sched_out(struct pe
 /*
  * Called with IRQs disabled
  */
-static void __perf_event_task_sched_out(struct perf_event_context *ctx)
-{
-	task_ctx_sched_out(ctx, EVENT_ALL);
-}
-
-/*
- * Called with IRQs disabled
- */
 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 			      enum event_type_t event_type)
 {
@@ -1434,7 +1426,7 @@ void perf_event_context_sched_in(struct 
  * accessing the event control register. If a NMI hits, then it will
  * keep the event running.
  */
-void perf_event_task_sched_in(struct task_struct *task)
+void __perf_event_task_sched_in(struct task_struct *task)
 {
 	struct perf_event_context *ctx;
 	int ctxn;
@@ -2150,7 +2142,8 @@ static void free_event(struct perf_event
 	irq_work_sync(&event->pending);
 
 	if (!event->parent) {
-		atomic_dec(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_dec(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_dec(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5286,7 +5279,8 @@ perf_event_alloc(struct perf_event_attr 
 	event->pmu = pmu;
 
 	if (!event->parent) {
-		atomic_inc(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_inc(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_inc(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5781,7 +5775,7 @@ static void perf_event_exit_task_context
 	 * our context.
 	 */
 	child_ctx = child->perf_event_ctxp[ctxn];
-	__perf_event_task_sched_out(child_ctx);
+	task_ctx_sched_out(child_ctx, EVENT_ALL);
 
 	/*
 	 * Take the context lock here so that if find_get_context is
#1 lore raw

RFC PATCH 1/7 perf: Fix task refcount issues

In reply to Peter Zijlstra

Currently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.

Cc Frederic Weisbecker <[email protected]>
Cc Matt Helsley <[email protected]>
---
 kernel/perf_event.c |    7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
kernel/perf_event.c +4 -3
Index: linux-2.6/kernel/perf_event.c
===================================================================
@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);
#11 lore raw

RFC PATCH 4/7 jump_label: More consitent naming

In reply to Peter Zijlstra

Now that there's only a few users around, rename things.

---
 include/linux/jump_label.h |    8 ++++----
 kernel/tracepoint.c        |    6 +++---
 lib/dynamic_debug.c        |    4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/jump_label.h +4 -4
Index: linux-2.6/include/linux/jump_label.h
===================================================================
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned l
 extern void jump_label_apply_nops(struct module *mod);
 extern int jump_label_text_reserved(void *start, void *end);
 
-#define enable_jump_label(key) \
+#define jump_label_enable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
 
-#define disable_jump_label(key) \
+#define jump_label_disable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
 
 #else
@@ -39,12 +39,12 @@ do {						\
 		goto label;			\
 } while (0)
 
-#define enable_jump_label(cond_var)	\
+#define jump_label_enable(cond_var)	\
 do {					\
        *(cond_var) = 1;			\
 } while (0)
 
-#define disable_jump_label(cond_var)	\
+#define jump_label_disable(cond_var)	\
 do {					\
        *(cond_var) = 0;			\
 } while (0)
kernel/tracepoint.c +3 -3
Index: linux-2.6/kernel/tracepoint.c
===================================================================
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracep
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
 	if (!elem->state && active) {
-		enable_jump_label(&elem->state);
+		jump_label_enable(&elem->state);
 		elem->state = active;
 	} else if (elem->state && !active) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = active;
 	}
 }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tr
 		elem->unregfunc();
 
 	if (elem->state) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = 0;
 	}
 	rcu_assign_pointer(elem->funcs, NULL);
lib/dynamic_debug.c +2 -2
Index: linux-2.6/lib/dynamic_debug.c
===================================================================
@@ -142,9 +142,9 @@ static void ddebug_change(const struct d
 				dt->num_enabled++;
 			dp->flags = newflags;
 			if (newflags) {
-				enable_jump_label(&dp->enabled);
+				jump_label_enable(&dp->enabled);
 			} else {
-				disable_jump_label(&dp->enabled);
+				jump_label_disable(&dp->enabled);
 			}
 			if (verbose)
 				printk(KERN_INFO
#13 lore raw

RFC PATCH 5/7 jump_label: atomic_t interface

In reply to Peter Zijlstra

Add an interface to allow usage of jump_labels with atomic counters

---
 include/linux/jump_label_ref.h |   44 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
include/linux/jump_label_ref.h new file +44 -0
Index: linux-2.6/include/linux/jump_label_ref.h
===================================================================
@@ -0,0 +1,44 @@
+#ifndef _LINUX_JUMP_LABEL_REF_H
+#define _LINUX_JUMP_LABEL_REF_H
+
+#include <linux/jump_label.h>
+#include <asm/atomic.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	if (atomic_add_return(1, key) == 1)
+		jump_label_enable(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	if (atomic_dec_and_test(key))
+		jump_label_disable(key);
+}
+
+#else /* !HAVE_JUMP_LABEL */
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	atomic_inc(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	atomic_dec(key);
+}
+
+#undef JUMP_LABEL
+#define JUMP_LABEL(key, label)						\
+do {									\
+	if (unlikely(__builtin_choose_expr(				\
+	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
+	      atomic_read((atomic_t *)(key)), *(key))))			\
+		goto label;						\
+} while (0)
+
+#endif /* HAVE_JUMP_LABEL */
+
+#endif /* _LINUX_JUMP_LABEL_REF_H */
maintainer
#22 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 18 lines · Peter Zijlstra

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +2 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);

include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028

Quoted 10 lines
include/linux/perf_event.h +7 -0
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 

OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’

The relevant snippet from the .i file reads:

static inline void
perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) {
struct pt_regs hot_regs;

do { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table, \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : : have_event); } while (0); return;

have_event:
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
}
__perf_sw_event(event_id, nr, nmi, regs, addr);
}

Anybody got any clue as to why this goes splat?

I tried both:

gcc (GCC) 4.4.4 20100630 (Red Hat 4.4.4-10)

and

x86_64-linux-gcc (GCC) 4.5.1

maintainer
#9 lore raw

Re: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

In reply to Frederic Weisbecker
Quoted 7 lines · Frederic Weisbecker
Quoted 1 line

+#define PERF_ATTACH_TASK 0x04

Thanks, the patch looks good.

I'm just not sure about the point of this flag...

Ah, see 6/7..

#23 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 57 lines · Peter Zijlstra
Quoted 18 lines

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +2 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);

include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028

Quoted 10 lines
include/linux/perf_event.h +7 -0
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 

OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’

The relevant snippet from the .i file reads:

static inline void
perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) {
struct pt_regs hot_regs;

do { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table, \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : : have_event); } while (0); return;

have_event:
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
}
__perf_sw_event(event_id, nr, nmi, regs, addr);
}

Quoted 2 lines

Anybody got any clue as to why this goes splat?

Couple issues here. The key value: "&perf_swevent_enabled[event_id]" is determined at run-time, so we can't associated the jump label with keys that are run-time dependent. The second thing is that there is array indexing, so essentially the jump label code would need to be smart enough to associated the correct index with each instance of 'perf_sw_event' - unfortunately its not that smart :(

So, two possible suggestions:

1)

I see that PERF_COUNT_SW_MAX is 9. So, we could explicitly split out the 9 cases into 9 separate inline functions where the keys would be:

&perf_swevent_enabled[PERF_COUNT_SW_CPU_CLOCK]
&perf_swevent_enabled[PERF_COUNT_SW_TASK_CLOCK]

this probably wouldn't look to ugly if there was a macro that took each software id, and spit out the appropriate inline. I see there aren't that many 'perf_sw_event()' calls either to update.

2)

If you could define a single global variable that was set if any of the indexes of the perf_swevent_enabled[] array were set. This potentially woulnd't be as efficient as #1 as it would create false positives.

thanks,

-Jason

maintainer
#24 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +2 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);

include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028

Quoted 10 lines
include/linux/perf_event.h +7 -0
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 

OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:

Quoted 5 lines

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’

Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.

I thought we had a __force_inline macro but I can't seem to find that.

Acked-by on v1 5/7
#14 lore raw

Re: [RFC][PATCH 5/7] jump_label: atomic_t interface

In reply to Peter Zijlstra
Quoted 21 lines · Peter Zijlstra

Add an interface to allow usage of jump_labels with atomic counters

---
 include/linux/jump_label_ref.h |   44 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

Index: linux-2.6/include/linux/jump_label_ref.h
===================================================================

include/linux/jump_label_ref.h new file +9 -0
@@ -0,0 +1,44 @@
+#ifndef _LINUX_JUMP_LABEL_REF_H
+#define _LINUX_JUMP_LABEL_REF_H
+
+#include <linux/jump_label.h>
+#include <asm/atomic.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static inline void jump_label_inc(atomic_t *key)

include/linux/jump_label_ref.h line 19

Quoted 10 lines
include/linux/jump_label_ref.h new file +10 -0
+{
+	if (atomic_add_return(1, key) == 1)
+		jump_label_enable(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	if (atomic_dec_and_test(key))
+		jump_label_disable(key);
+}

Nice. I was trying to find a more self-explanatory name, like jump_label_get/put or jump_label_inc_enable/jump_label_dec_disable.

But in fact the current name looks good eventually.

Quoted 26 lines
include/linux/jump_label_ref.h new file +24 -0
+#else /* !HAVE_JUMP_LABEL */
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	atomic_inc(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	atomic_dec(key);
+}
+
+#undef JUMP_LABEL
+#define JUMP_LABEL(key, label)						\
+do {									\
+	if (unlikely(__builtin_choose_expr(				\
+	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
+	      atomic_read((atomic_t *)(key)), *(key))))			\
+		goto label;						\
+} while (0)
+
+#endif /* HAVE_JUMP_LABEL */
+
+#endif /* _LINUX_JUMP_LABEL_REF_H */
#25 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 35 lines · Peter Zijlstra
Quoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +2 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);

include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028

Quoted 10 lines
include/linux/perf_event.h +7 -0
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 

OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:

Quoted 9 lines
Quoted 5 lines

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’

Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.

ah...cool! that should work. ignore my other suggestions...

thanks,

-Jason

#8 lore raw

Re: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

In reply to Peter Zijlstra
Quoted 39 lines · Peter Zijlstra

hw_breakpoint creation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.

With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.

This makes hw_breakpoints unhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in the event::hw bits for now...

Cc Frederic Weisbecker <[email protected]>
---
 include/linux/perf_event.h |    7 +++++++
 kernel/hw_breakpoint.c     |    8 ++++----
 kernel/perf_event.c        |   21 ++++++++++++++++-----
 3 files changed, 27 insertions(+), 9 deletions(-)

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +6 -0
@@ -536,6 +536,12 @@ struct hw_perf_event {
 		struct { /* breakpoint */
 			struct arch_hw_breakpoint	info;
 			struct list_head		bp_list;
+			/*
+			 * Crufty hack to avoid the chicken and egg
+			 * problem hw_breakpoint has with context
+			 * creation and event initalization.
+			 */
+			struct task_struct		*bp_target;
 		};
 #endif
 	};

include/linux/perf_event.h struct swevent_hlist { · line 702

Quoted 4 lines
include/linux/perf_event.h +1 -0
@@ -693,6 +699,7 @@ struct swevent_hlist {
 
 #define PERF_ATTACH_CONTEXT	0x01
 #define PERF_ATTACH_GROUP	0x02
+#define PERF_ATTACH_TASK	0x04

Thanks, the patch looks good.

I'm just not sure about the point of this flag...

Acked-by on v1 1/7
#2 lore raw

Re: [RFC][PATCH 1/7] perf: Fix task refcount issues

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra

Currently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.

Cc Frederic Weisbecker <[email protected]>
Cc Matt Helsley <[email protected]>

Looks good.

Quoted 41 lines
---
 kernel/perf_event.c |    7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

Index: linux-2.6/kernel/perf_event.c
===================================================================

kernel/perf_event.c +4 -3
@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);
maintainer
#26 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 4 lines · Peter Zijlstra

Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.

I thought we had a __force_inline macro but I can't seem to find that.

Do you mean __always_inline?

See include/linux/compiler-gcc4.h

-- Steve

maintainer
#27 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

Quoted 10 lines · Steven Rostedt
Quoted 4 lines · Peter Zijlstra

Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.

I thought we had a __force_inline macro but I can't seem to find that.

Do you mean __always_inline?

See include/linux/compiler-gcc4.h

Yeah, cyrillos already pointed that out..

Acked-by on v1 1/7
#3 lore raw

Re: [RFC][PATCH 1/7] perf: Fix task refcount issues

In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

Currently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.

Doh.

Looks good.

Quoted 49 lines
Cc Frederic Weisbecker <[email protected]>
Cc Matt Helsley <[email protected]>
---
 kernel/perf_event.c |    7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

Index: linux-2.6/kernel/perf_event.c
===================================================================

kernel/perf_event.c +4 -4
@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#28 lore raw

Re: [RFC][PATCH 7/7] perf: Optimize sw events

In reply to Peter Zijlstra
Quoted 41 lines · Peter Zijlstra
Quoted 34 lines · Peter Zijlstra
Quoted 18 lines · Peter Zijlstra

Index: linux-2.6/include/linux/perf_event.h
===================================================================

include/linux/perf_event.h +2 -7
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);

include/linux/perf_event.h static inline void perf_fetch_caller_reg · line 1028

Quoted 10 lines
include/linux/perf_event.h +7 -0
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 

OK, so it appears I only compile tested this bit without jump_label support, with that bit added back this horribly fails to compile like:

Quoted 5 lines

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’

Quoted 4 lines

Ah, figured it out, it needs __attribute__((always_inline)), otherwise we end up with multiple keys for one label, which will clealy not work.

I thought we had a __force_inline macro but I can't seem to find that.

It's called __always_inline.

Thanks,

Ingo

maintainer
#19 lore raw

Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

In reply to Peter Zijlstra

include/linux/perf_event.h struct perf_output_handle { · line 909

Quoted 8 lines · Peter Zijlstra
include/linux/perf_event.h +8 -0
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+       JUMP_LABEL(&perf_task_events, have_events);
+       return;
+
+have_events:
+       __perf_event_task_sched_in(task);
+} 

OK, so I hate the JUMP_LABEL() interface for it means we have to keep writing these silly stubs.

How about something like:

#define COND_STMT(key, stmt) \

do {								\
	__label__ jl_enabled;					\
	JUMP_LABEL(key, jl_enabled);				\
	if (0) {						\
jl_enabled:							\
		stmt;						\
	}							\
} while (0)
maintainer
#20 lore raw

Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

Quoted 12 lines · Peter Zijlstra

Review

Quoted 8 lines · Peter Zijlstra
(quoted diff) +8 -0
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+       JUMP_LABEL(&perf_task_events, have_events);
+       return;
+
+have_events:
+       __perf_event_task_sched_in(task);
+} 

OK, so I hate the JUMP_LABEL() interface for it means we have to keep writing these silly stubs.

The below seems to actually compile too, awesome! ;-)

---

include/linux/jump_label.h +10 -0
Index: linux-2.6/include/linux/jump_label.h
===================================================================
@@ -61,4 +61,14 @@ static inline int jump_label_text_reserv
 
 #endif
 
+#define COND_STMT(key, stmt)					\
+do {								\
+	__label__ jl_enabled;					\
+	JUMP_LABEL(key, jl_enabled);				\
+	if (0) {						\
+jl_enabled:							\
+		stmt;						\
+	}							\
+} while (0)
+
 #endif
include/linux/perf_event.h +2 -10
Index: linux-2.6/include/linux/perf_event.h
===================================================================
@@ -901,21 +901,13 @@ extern atomic_t perf_task_events;
 
 static inline void perf_event_task_sched_in(struct task_struct *task)
 {
-	JUMP_LABEL(&perf_task_events, have_events);
-	return;
-
-have_events:
-	__perf_event_task_sched_in(task);
+	COND_STMT(&perf_task_events, __perf_event_task_sched_in(task));
 }
 
 static inline
 void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
 {
-	JUMP_LABEL(&perf_task_events, have_events);
-	return;
-
-have_events:
-	__perf_event_task_sched_out(task, next);
+	COND_STMT(&perf_task_events, __perf_event_task_sched_out(task, next));
 }
 
 extern int perf_event_init_task(struct task_struct *child);
#4 lore raw

[tip:perf/core] perf: Fix task refcount bugs

In reply to Peter Zijlstra
Commit-ID:  e7d0bc047548d76feee6b23f7d3d9da927189a50
Gitweb:     http://git.kernel.org/tip/e7d0bc047548d76feee6b23f7d3d9da927189a50
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 16:54:51 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:52 +0200

perf: Fix task refcount bugs

Currently it looks like find_lively_task_by_vpid() takes a task ref and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be dropping task refs.

LKML-Reference: <[email protected]>

---
 kernel/perf_event.c |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)
kernel/perf_event.c +4 -3
index 346dc0e..f928878 100644
@@ -2190,11 +2190,9 @@ retry:
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5602,7 +5600,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5698,6 +5696,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);
#6 lore raw

[tip:perf/core] perf: Find task before event alloc

In reply to Peter Zijlstra
Commit-ID:  c6be5a5cb62592d9d661899a2aa78236eb00ffa5
Gitweb:     http://git.kernel.org/tip/c6be5a5cb62592d9d661899a2aa78236eb00ffa5
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 16:59:46 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:54 +0200

perf: Find task before event alloc

So that we can pass the task pointer to the event allocation, so that we can use task associated data during event initialization.

LKML-Reference: <[email protected]>

---
 kernel/perf_event.c |   23 ++++++++++++-----------
 1 files changed, 12 insertions(+), 11 deletions(-)
kernel/perf_event.c +12 -11
index f928878..b21d06a 100644
@@ -5551,10 +5551,18 @@ SYSCALL_DEFINE5(perf_event_open,
 			group_leader = NULL;
 	}
 
+	if (pid != -1) {
+		task = find_lively_task_by_vpid(pid);
+		if (IS_ERR(task)) {
+			err = PTR_ERR(task);
+			goto err_group_fd;
+		}
+	}
+
 	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
-		goto err_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5586,21 +5594,13 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	if (pid != -1) {
-		task = find_lively_task_by_vpid(pid);
-		if (IS_ERR(task)) {
-			err = PTR_ERR(task);
-			goto err_group_fd;
-		}
-	}
-
 	/*
 	 * Get the target context (task or percpu):
 	 */
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_task;
+		goto err_alloc;
 	}
 
 	/*
@@ -5696,12 +5696,13 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_alloc:
+	free_event(event);
 err_task:
 	if (task)
 		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
-	free_event(event);
 err_fd:
 	put_unused_fd(event_fd);
 	return err;
#10 lore raw

[tip:perf/core] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

In reply to Peter Zijlstra
Commit-ID:  d580ff8699e8811a9af37e9de4dea375401bdeec
Gitweb:     http://git.kernel.org/tip/d580ff8699e8811a9af37e9de4dea375401bdeec
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 17:43:23 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:55 +0200

perf, hw_breakpoint: Fix crash in hw_breakpoint creation

hw_breakpoint creation needs to account stuff per-task to ensure there is always sufficient hardware resources to back these things due to ptrace.

With the perf per pmu context changes the event initialization no longer has access to the event context, for the simple reason that we need to first find the pmu (result of initialization) before we can find the context.

This makes hw_breakpoints unhappy, because it can no longer do per task accounting, cure this by frobbing a task pointer in the event::hw bits for now...

Cc Frederic Weisbecker <[email protected]>

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |    7 +++++++
 kernel/hw_breakpoint.c     |    8 ++++----
 kernel/perf_event.c        |   23 ++++++++++++++++++-----
 3 files changed, 29 insertions(+), 9 deletions(-)
include/linux/perf_event.h +7 -0
index 2ebfc9a..97965fa 100644
@@ -536,6 +536,12 @@ struct hw_perf_event {
 		struct { /* breakpoint */
 			struct arch_hw_breakpoint	info;
 			struct list_head		bp_list;
+			/*
+			 * Crufty hack to avoid the chicken and egg
+			 * problem hw_breakpoint has with context
+			 * creation and event initalization.
+			 */
+			struct task_struct		*bp_target;
 		};
 #endif
 	};
@@ -693,6 +699,7 @@ struct swevent_hlist {
 
 #define PERF_ATTACH_CONTEXT	0x01
 #define PERF_ATTACH_GROUP	0x02
+#define PERF_ATTACH_TASK	0x04
 
 /**
  * struct perf_event - performance event kernel representation:
kernel/hw_breakpoint.c +4 -4
index 3b714e8..2c9120f 100644
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
  */
 static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
 {
-	struct perf_event_context *ctx = bp->ctx;
+	struct task_struct *tsk = bp->hw.bp_target;
 	struct perf_event *iter;
 	int count = 0;
 
 	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-		if (iter->ctx == ctx && find_slot_idx(iter) == type)
+		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
 			count += hw_breakpoint_weight(iter);
 	}
 
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
 		    enum bp_type_idx type)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	if (cpu >= 0) {
 		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
 	       int weight)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	/* Pinned counter cpu profiling */
 	if (!tsk) {
kernel/perf_event.c +18 -5
index b21d06a..856e20b 100644
@@ -5255,9 +5255,10 @@ unlock:
  */
 static struct perf_event *
 perf_event_alloc(struct perf_event_attr *attr, int cpu,
-		   struct perf_event *group_leader,
-		   struct perf_event *parent_event,
-		   perf_overflow_handler_t overflow_handler)
+		 struct task_struct *task,
+		 struct perf_event *group_leader,
+		 struct perf_event *parent_event,
+		 perf_overflow_handler_t overflow_handler)
 {
 	struct pmu *pmu;
 	struct perf_event *event;
@@ -5299,6 +5300,17 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
 
 	event->state		= PERF_EVENT_STATE_INACTIVE;
 
+	if (task) {
+		event->attach_state = PERF_ATTACH_TASK;
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+		/*
+		 * hw_breakpoint is a bit difficult here..
+		 */
+		if (attr->type == PERF_TYPE_BREAKPOINT)
+			event->hw.bp_target = task;
+#endif
+	}
+
 	if (!overflow_handler && parent_event)
 		overflow_handler = parent_event->overflow_handler;
 	
@@ -5559,7 +5571,7 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
+	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err_task;
@@ -5728,7 +5740,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
 	 * Get the target context (task or percpu):
 	 */
 
-	event = perf_event_alloc(attr, cpu, NULL, NULL, overflow_handler);
+	event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err;
@@ -5996,6 +6008,7 @@ inherit_event(struct perf_event *parent_event,
 
 	child_event = perf_event_alloc(&parent_event->attr,
 					   parent_event->cpu,
+					   child,
 					   group_leader, parent_event,
 					   NULL);
 	if (IS_ERR(child_event))
#12 lore raw

[tip:perf/core] jump_label: Use more consistent naming

In reply to Peter Zijlstra
Commit-ID:  3b6e901f839f42afb40f614418df82c08b01320a
Gitweb:     http://git.kernel.org/tip/3b6e901f839f42afb40f614418df82c08b01320a
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 21:10:38 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:56 +0200

jump_label: Use more consistent naming

Now that there's still only a few users around, rename things to make them more consistent.

LKML-Reference: <[email protected]>

---
 include/linux/jump_label.h |    8 ++++----
 kernel/tracepoint.c        |    6 +++---
 lib/dynamic_debug.c        |    4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/jump_label.h +4 -4
index b72cd9f..81be496 100644
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned long key, enum jump_label_type type);
 extern void jump_label_apply_nops(struct module *mod);
 extern int jump_label_text_reserved(void *start, void *end);
 
-#define enable_jump_label(key) \
+#define jump_label_enable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
 
-#define disable_jump_label(key) \
+#define jump_label_disable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
 
 #else
@@ -39,12 +39,12 @@ do {						\
 		goto label;			\
 } while (0)
 
-#define enable_jump_label(cond_var)	\
+#define jump_label_enable(cond_var)	\
 do {					\
        *(cond_var) = 1;			\
 } while (0)
 
-#define disable_jump_label(cond_var)	\
+#define jump_label_disable(cond_var)	\
 do {					\
        *(cond_var) = 0;			\
 } while (0)
kernel/tracepoint.c +3 -3
index d6073a5..e95ee7f 100644
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
 	if (!elem->state && active) {
-		enable_jump_label(&elem->state);
+		jump_label_enable(&elem->state);
 		elem->state = active;
 	} else if (elem->state && !active) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = active;
 	}
 }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tracepoint *elem)
 		elem->unregfunc();
 
 	if (elem->state) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = 0;
 	}
 	rcu_assign_pointer(elem->funcs, NULL);
lib/dynamic_debug.c +2 -2
index e925c7b..7bd6df7 100644
@@ -142,9 +142,9 @@ static void ddebug_change(const struct ddebug_query *query,
 				dt->num_enabled++;
 			dp->flags = newflags;
 			if (newflags) {
-				enable_jump_label(&dp->enabled);
+				jump_label_enable(&dp->enabled);
 			} else {
-				disable_jump_label(&dp->enabled);
+				jump_label_disable(&dp->enabled);
 			}
 			if (verbose)
 				printk(KERN_INFO
#15 lore raw

[tip:perf/core] jump_label: Add atomic_t interface

In reply to Peter Zijlstra
Commit-ID:  8b92538d84e50062560ba33adbaed7887b6e4a42
Gitweb:     http://git.kernel.org/tip/8b92538d84e50062560ba33adbaed7887b6e4a42
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 21:39:02 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:57 +0200

jump_label: Add atomic_t interface

Add an interface to allow usage of jump_labels with atomic counters.

LKML-Reference: <[email protected]>

---
 include/linux/jump_label_ref.h |   44 ++++++++++++++++++++++++++++++++++++++++
 1 files changed, 44 insertions(+), 0 deletions(-)
include/linux/jump_label_ref.h new file +44 -0
new file mode 100644
index 0000000..e5d012a
@@ -0,0 +1,44 @@
+#ifndef _LINUX_JUMP_LABEL_REF_H
+#define _LINUX_JUMP_LABEL_REF_H
+
+#include <linux/jump_label.h>
+#include <asm/atomic.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	if (atomic_add_return(1, key) == 1)
+		jump_label_enable(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	if (atomic_dec_and_test(key))
+		jump_label_disable(key);
+}
+
+#else /* !HAVE_JUMP_LABEL */
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	atomic_inc(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	atomic_dec(key);
+}
+
+#undef JUMP_LABEL
+#define JUMP_LABEL(key, label)						\
+do {									\
+	if (unlikely(__builtin_choose_expr(				\
+	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
+	      atomic_read((atomic_t *)(key)), *(key))))			\
+		goto label;						\
+} while (0)
+
+#endif /* HAVE_JUMP_LABEL */
+
+#endif /* _LINUX_JUMP_LABEL_REF_H */
#16 lore raw

[tip:perf/core] perf: Use jump_labels to optimize the scheduler hooks

In reply to Peter Zijlstra
Commit-ID:  82cd6def9806dcb6a325fb6abbc1d61388a15f6a
Gitweb:     http://git.kernel.org/tip/82cd6def9806dcb6a325fb6abbc1d61388a15f6a
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 14 Oct 2010 17:57:23 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 18 Oct 2010 19:58:58 +0200

perf: Use jump_labels to optimize the scheduler hooks

Trades a call + conditional + ret for an unconditional jmp.

LKML-Reference: <[email protected]>

---
 include/linux/perf_event.h |   27 +++++++++++++++++++++++++--
 kernel/perf_event.c        |   24 +++++++++---------------
 2 files changed, 34 insertions(+), 17 deletions(-)
include/linux/perf_event.h +25 -2
index 97965fa..7f0e7f5 100644
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/irq_work.h>
+#include <linux/jump_label_ref.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
 
@@ -895,8 +896,30 @@ extern void perf_pmu_unregister(struct pmu *pmu);
 
 extern int perf_num_counters(void);
 extern const char *perf_pmu_name(void);
-extern void perf_event_task_sched_in(struct task_struct *task);
-extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+extern void __perf_event_task_sched_in(struct task_struct *task);
+extern void __perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+
+extern atomic_t perf_task_events;
+
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_in(task);
+}
+
+static inline
+void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_out(task, next);
+}
+
 extern int perf_event_init_task(struct task_struct *child);
 extern void perf_event_exit_task(struct task_struct *child);
 extern void perf_event_free_task(struct task_struct *task);
kernel/perf_event.c +9 -15
index 856e20b..f7febb0 100644
@@ -34,7 +34,7 @@
 
 #include <asm/irq_regs.h>
 
-static atomic_t nr_events __read_mostly;
+atomic_t perf_task_events __read_mostly;
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
@@ -1311,8 +1311,8 @@ void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  * accessing the event control register. If a NMI hits, then it will
  * not restart the event.
  */
-void perf_event_task_sched_out(struct task_struct *task,
-			       struct task_struct *next)
+void __perf_event_task_sched_out(struct task_struct *task,
+				 struct task_struct *next)
 {
 	int ctxn;
 
@@ -1340,14 +1340,6 @@ static void task_ctx_sched_out(struct perf_event_context *ctx,
 /*
  * Called with IRQs disabled
  */
-static void __perf_event_task_sched_out(struct perf_event_context *ctx)
-{
-	task_ctx_sched_out(ctx, EVENT_ALL);
-}
-
-/*
- * Called with IRQs disabled
- */
 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 			      enum event_type_t event_type)
 {
@@ -1494,7 +1486,7 @@ void perf_event_context_sched_in(struct perf_event_context *ctx)
  * accessing the event control register. If a NMI hits, then it will
  * keep the event running.
  */
-void perf_event_task_sched_in(struct task_struct *task)
+void __perf_event_task_sched_in(struct task_struct *task)
 {
 	struct perf_event_context *ctx;
 	int ctxn;
@@ -2216,7 +2208,8 @@ static void free_event(struct perf_event *event)
 	irq_work_sync(&event->pending);
 
 	if (!event->parent) {
-		atomic_dec(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_dec(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_dec(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5354,7 +5347,8 @@ done:
 	event->pmu = pmu;
 
 	if (!event->parent) {
-		atomic_inc(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_inc(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_inc(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5849,7 +5843,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
 	 * our context.
 	 */
 	child_ctx = child->perf_event_ctxp[ctxn];
-	__perf_event_task_sched_out(child_ctx);
+	task_ctx_sched_out(child_ctx, EVENT_ALL);
 
 	/*
 	 * Take the context lock here so that if find_get_context is