generic software counters -v2
Subsystems (4)
cover generic software counters -v2
This patch set improves the current software counters by providing sampling events for most of them.
The clock based software counters use hrtimers to generate events, allowing sample based profiling without hardware support -- obviously this has limited functionality for kernal based profiling since we disable interrupts quite frequently, but should be good for userspace.
Other events like pagefaults insert simple hooks into the actual pagefault code to generate these events.
Future work (in-progress) will hook into the tracepoint infrastructure to to provide profiling events.
PATCH 1/11 sched: remove extra call overhead for schedule()
Lai Jiangshan's patch reminded me that I promised Nick to remove that extra call overhead in schedule().
--- kernel/mutex.c | 4 +++- kernel/sched.c | 12 ++++-------- 2 files changed, 7 insertions(+), 9 deletions(-)
kernel/mutex.c +3 -1
@@ -248,7 +248,9 @@ __mutex_lock_common(struct mutex *lock,/* didnt get the lock, go to sleep: */ spin_unlock_mutex (&lock->wait_lock, flags); -__schedule (); +preempt_enable_no_resched (); +schedule (); +preempt_disable ();spin_lock_mutex (&lock->wait_lock, flags); }
kernel/sched.c +4 -8
@@ -4763,13 +4763,15 @@ pick_next_task(struct rq *rq)/* * schedule() is the main scheduler function. */ -asmlinkage void __sched __schedule(void) +asmlinkage void __sched schedule(void) {struct task_struct *prev, *next;unsigned long *switch_count;struct rq *rq;int cpu; +need_resched: +preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_qsctr_inc (cpu);
@@ -4827,15 +4829,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; -} -asmlinkage void __sched schedule(void) -{ -need_resched: -preempt_disable (); -__schedule ();preempt_enable_no_resched (); -if (unlikely (test_thread_flag (TIF_NEED_RESCHED ))) +if (need_resched ()) goto need_resched; }EXPORT_SYMBOL (schedule);
[tip:core/locking] sched: remove extra call overhead for schedule()
Commit-ID: e690e5337d953a0936a9735424a454ec21bae0a5 Gitweb: http://git.kernel.org/tip/e690e5337d953a0936a9735424a454ec21bae0a5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:26 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:55:17 +0100
sched: remove extra call overhead for schedule()
Lai Jiangshan's patch reminded me that I promised Nick to remove that extra call overhead in schedule().
LKML-Reference: <[email protected]>
--- kernel/mutex.c | 4 +++- kernel/sched.c | 12 ++++-------- 2 files changed, 7 insertions(+), 9 deletions(-)
kernel/mutex.c +3 -1
@@ -248,7 +248,9 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,/* didnt get the lock, go to sleep: */ spin_unlock_mutex (&lock->wait_lock, flags); -__schedule (); +preempt_enable_no_resched (); +schedule (); +preempt_disable ();spin_lock_mutex (&lock->wait_lock, flags); }
kernel/sched.c +4 -8
@@ -4543,13 +4543,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev)/* * schedule() is the main scheduler function. */ -asmlinkage void __sched __schedule(void) +asmlinkage void __sched schedule(void) {struct task_struct *prev, *next;unsigned long *switch_count;struct rq *rq;int cpu; +need_resched: +preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_qsctr_inc (cpu);
@@ -4606,15 +4608,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; -} -asmlinkage void __sched schedule(void) -{ -need_resched: -preempt_disable (); -__schedule ();preempt_enable_no_resched (); -if (unlikely (test_thread_flag (TIF_NEED_RESCHED ))) +if (need_resched ()) goto need_resched; }EXPORT_SYMBOL (schedule);
[tip:sched/core] sched: remove extra call overhead for schedule()
In reply to Peter ZijlstraCommit-ID: ff743345bf7685a207868048a70e23164c4785e5 Gitweb: http://git.kernel.org/tip/ff743345bf7685a207868048a70e23164c4785e5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:26 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 20 Apr 2009 20:49:53 +0200
sched: remove extra call overhead for schedule()
Lai Jiangshan's patch reminded me that I promised Nick to remove that extra call overhead in schedule().
LKML-Reference: <[email protected]>
--- kernel/mutex.c | 4 +++- kernel/sched.c | 12 ++++-------- 2 files changed, 7 insertions(+), 9 deletions(-)
kernel/mutex.c +3 -1
@@ -248,7 +248,9 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,/* didnt get the lock, go to sleep: */ spin_unlock_mutex (&lock->wait_lock, flags); -__schedule (); +preempt_enable_no_resched (); +schedule (); +preempt_disable ();spin_lock_mutex (&lock->wait_lock, flags); }
kernel/sched.c +4 -8
@@ -5131,13 +5131,15 @@ pick_next_task(struct rq *rq)/* * schedule() is the main scheduler function. */ -asmlinkage void __sched __schedule(void) +asmlinkage void __sched schedule(void) {struct task_struct *prev, *next;unsigned long *switch_count;struct rq *rq;int cpu; +need_resched: +preempt_disable (); cpu =smp_processor_id (); rq =cpu_rq (cpu);rcu_qsctr_inc (cpu);
@@ -5194,15 +5196,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; -} -asmlinkage void __sched schedule(void) -{ -need_resched: -preempt_disable (); -__schedule ();preempt_enable_no_resched (); -if (unlikely (test_thread_flag (TIF_NEED_RESCHED ))) +if (need_resched ()) goto need_resched; }EXPORT_SYMBOL (schedule);
PATCH 2/11 hrtimer: fix rq->lock inversion (again)
In reply to Peter ZijlstraIt appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
--- include/linux/interrupt.h | 1 kernel/hrtimer.c | 55 ++++++++++++++++++++++++++++------------------ kernel/sched.c | 17 +++++++++++--- kernel/softirq.c | 2 - 4 files changed, 50 insertions(+), 25 deletions(-)
kernel/hrtimer.c +34 -21
@@ -651,14 +651,20 @@ static inline void hrtimer_init_timer_hr* and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram (struct hrtimer *timer, -struct hrtimer_clock_base *base) +struct hrtimer_clock_base *base, +int wakeup) {if (base->cpu_base->hres_active &&hrtimer_reprogram (timer, base)) { -spin_unlock (&base->cpu_base->lock); -raise_softirq_irqoff (HRTIMER_SOFTIRQ ); -spin_lock (&base->cpu_base->lock); +if (wakeup) { +spin_unlock (&base->cpu_base->lock); +raise_softirq_irqoff (HRTIMER_SOFTIRQ ); +spin_lock (&base->cpu_base->lock); + }else +__raise_softirq_irqoff (HRTIMER_SOFTIRQ ); +return 1 ; } +return 0 ; }
@@ -703,7 +709,8 @@ static inline int hrtimer_is_hres_enablestatic inline int hrtimer_switch_to_hres (void ) {return 0 ; }static inline void hrtimer_force_reprogram (struct hrtimer_cpu_base *base) { }static inline int hrtimer_enqueue_reprogram (struct hrtimer *timer, -struct hrtimer_clock_base *base) +struct hrtimer_clock_base *base, +int wakeup) {return 0 ; }
@@ -886,20 +893,9 @@ remove_hrtimer(struct hrtimer *timer, streturn 0 ; } -/** -* hrtimer_start_range_ns - (re)start an hrtimer on the current CPU -* @timer: the timer to be added -* @tim: expiry time -* @delta_ns: "slack" range for the timer -* @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) -* -* Returns: -* 0 on success -* 1 when the timer was active -*/ -int -hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim,unsigned long delta_ns, -const enum hrtimer_mode mode) +int __hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns,const enum hrtimer_mode mode, +int wakeup) {struct hrtimer_clock_base *base, *new_base;unsigned long flags;
@@ -940,12 +936,29 @@ hrtimer_start_range_ns(struct hrtimer *t* XXX send_remote_softirq() ? */ if (leftmost && new_base->cpu_base == &__get_cpu_var (hrtimer_bases)) -hrtimer_enqueue_reprogram (timer, new_base); +hrtimer_enqueue_reprogram (timer, new_base, wakeup);unlock_hrtimer_base (timer, &flags);return ret; } + +/** +* hrtimer_start_range_ns - (re)start an hrtimer on the current CPU +* @timer: the timer to be added +* @tim: expiry time +* @delta_ns: "slack" range for the timer +* @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) +* +* Returns: +* 0 on success +* 1 when the timer was active +*/ +int hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns,const enum hrtimer_mode mode) +{ +return __hrtimer_start_range_ns (timer, tim, delta_ns, mode,1 ); +}EXPORT_SYMBOL_GPL (hrtimer_start_range_ns);/**
@@ -961,7 +974,7 @@ EXPORT_SYMBOL_GPL(hrtimer_start_range_nsint hrtimer_start (struct hrtimer *timer,ktime_t tim,const enum hrtimer_mode mode) { -return hrtimer_start_range_ns (timer, tim,0 , mode); +return __hrtimer_start_range_ns (timer, tim,0 , mode,1 ); }EXPORT_SYMBOL_GPL (hrtimer_start);
kernel/sched.c +14 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rtspin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1170,7 +1177,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
@@ -4830,6 +4838,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; +if (unlikely (local_softirq_pending ())) +wakeup_softirqd (); +preempt_enable_no_resched ();if (need_resched ()) goto need_resched;
include/linux/interrupt.h +1 -0
@@ -274,6 +274,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/softirq.c +1 -1
@@ -59,7 +59,7 @@ static DEFINE_PER_CPU(struct task_struct* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
[tip:core/locking] hrtimer: fix rq->lock inversion (again)
Commit-ID: 75f9078edfebb697647ba00c1bb3df533fb8b713 Gitweb: http://git.kernel.org/tip/75f9078edfebb697647ba00c1bb3df533fb8b713 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:27 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:55:18 +0100
hrtimer: fix rq->lock inversion (again)
It appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
LKML-Reference: <[email protected]>
--- include/linux/interrupt.h | 1 + kernel/hrtimer.c | 55 +++++++++++++++++++++++++++----------------- kernel/sched.c | 17 +++++++++++-- kernel/softirq.c | 2 +- 4 files changed, 50 insertions(+), 25 deletions(-)
include/linux/interrupt.h +1 -0
@@ -274,6 +274,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/hrtimer.c +34 -21
94 lines not on this page · raw message
kernel/sched.c +14 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)spin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1129,7 +1136,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
@@ -4609,6 +4617,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; +if (unlikely (local_softirq_pending ())) +wakeup_softirqd (); +preempt_enable_no_resched ();if (need_resched ()) goto need_resched;
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
[tip:core/locking] hrtimer: fix rq->lock inversion (again)
In reply to Peter ZijlstraCommit-ID: c2038b4dcea7c93bb53b0224754c126b7960a1b5 Gitweb: http://git.kernel.org/tip/c2038b4dcea7c93bb53b0224754c126b7960a1b5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:27 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:17:55 +0100
hrtimer: fix rq->lock inversion (again)
It appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
LKML-Reference: <[email protected]>
--- include/linux/hrtimer.h | 5 ++++ include/linux/interrupt.h | 1 + kernel/hrtimer.c | 55 +++++++++++++++++++++++++++----------------- kernel/sched.c | 17 +++++++++++-- kernel/softirq.c | 2 +- 5 files changed, 55 insertions(+), 25 deletions(-)
include/linux/hrtimer.h +5 -0
@@ -336,6 +336,11 @@ extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode);extern int hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim,unsigned long range_ns,const enum hrtimer_mode mode); +extern int +__hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns, +const enum hrtimer_mode mode,int wakeup); +extern int hrtimer_cancel (struct hrtimer *timer);extern int hrtimer_try_to_cancel (struct hrtimer *timer);
include/linux/interrupt.h +1 -0
@@ -274,6 +274,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/hrtimer.c +34 -21
94 lines not on this page · raw message
kernel/sched.c +14 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)spin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1129,7 +1136,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
@@ -4609,6 +4617,9 @@ need_resched_nonpreemptible:if (unlikely (reacquire_kernel_lock (current) <0 )) goto need_resched_nonpreemptible; +if (unlikely (local_softirq_pending ())) +wakeup_softirqd (); +preempt_enable_no_resched ();if (need_resched ()) goto need_resched;
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
[tip:core/locking] hrtimer: fix rq->lock inversion (again)
In reply to Peter ZijlstraCommit-ID: 84648567ee357abd8879257018f38f8b6b35334d Gitweb: http://git.kernel.org/tip/84648567ee357abd8879257018f38f8b6b35334d Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:27 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 15:31:28 +0100
hrtimer: fix rq->lock inversion (again)
It appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
LKML-Reference: <[email protected]>
--- include/linux/hrtimer.h | 5 ++++ include/linux/interrupt.h | 1 + kernel/hrtimer.c | 55 +++++++++++++++++++++++++++----------------- kernel/sched.c | 14 +++++++++-- kernel/softirq.c | 2 +- 5 files changed, 52 insertions(+), 25 deletions(-)
include/linux/hrtimer.h +5 -0
@@ -336,6 +336,11 @@ extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode);extern int hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim,unsigned long range_ns,const enum hrtimer_mode mode); +extern int +__hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns, +const enum hrtimer_mode mode,int wakeup); +extern int hrtimer_cancel (struct hrtimer *timer);extern int hrtimer_try_to_cancel (struct hrtimer *timer);
include/linux/interrupt.h +1 -0
@@ -274,6 +274,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/hrtimer.c +34 -21
94 lines not on this page · raw message
kernel/sched.c +11 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)spin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1129,7 +1136,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
[tip:core/locking] hrtimer: fix rq->lock inversion (again)
In reply to Peter ZijlstraCommit-ID: 7f1e2ca9f04b02794597f60e7b1d43f0a1317939 Gitweb: http://git.kernel.org/tip/7f1e2ca9f04b02794597f60e7b1d43f0a1317939 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:27 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 31 Mar 2009 14:52:52 +0200
hrtimer: fix rq->lock inversion (again)
It appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
LKML-Reference: <[email protected]>
--- include/linux/hrtimer.h | 5 ++++ include/linux/interrupt.h | 1 + kernel/hrtimer.c | 55 +++++++++++++++++++++++++++----------------- kernel/sched.c | 14 +++++++++-- kernel/softirq.c | 2 +- 5 files changed, 52 insertions(+), 25 deletions(-)
include/linux/hrtimer.h +5 -0
@@ -336,6 +336,11 @@ extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode);extern int hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim,unsigned long range_ns,const enum hrtimer_mode mode); +extern int +__hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns, +const enum hrtimer_mode mode,int wakeup); +extern int hrtimer_cancel (struct hrtimer *timer);extern int hrtimer_try_to_cancel (struct hrtimer *timer);
include/linux/interrupt.h +1 -0
@@ -294,6 +294,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/hrtimer.c +34 -21
94 lines not on this page · raw message
kernel/sched.c +11 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)spin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1146,7 +1153,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
[tip:timers/urgent] hrtimer: fix rq->lock inversion (again)
In reply to Peter ZijlstraCommit-ID: 4f70f0be7dd540085b5ac54e1191e212fae1478d Gitweb: http://git.kernel.org/tip/4f70f0be7dd540085b5ac54e1191e212fae1478d Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:27 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 2 Apr 2009 18:26:13 +0200
hrtimer: fix rq->lock inversion (again)
It appears I inadvertly introduced rq->lock recursion to the hrtimer_start() path when I delegated running already expired timers to softirq context.
This patch fixes it by introducing a __hrtimer_start_range_ns() method that will not use raise_softirq_irqoff() but __raise_softirq_irqoff() which avoids the wakeup.
It then also changes schedule() to check for pending softirqs and do the wakeup then, I'm not quite sure I like this last bit, nor am I convinced its really needed.
LKML-Reference: <[email protected]>
--- include/linux/hrtimer.h | 5 ++++ include/linux/interrupt.h | 1 + kernel/hrtimer.c | 55 +++++++++++++++++++++++++++----------------- kernel/sched.c | 14 +++++++++-- kernel/softirq.c | 2 +- 5 files changed, 52 insertions(+), 25 deletions(-)
include/linux/hrtimer.h +5 -0
@@ -336,6 +336,11 @@ extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode);extern int hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim,unsigned long range_ns,const enum hrtimer_mode mode); +extern int +__hrtimer_start_range_ns (struct hrtimer *timer,ktime_t tim, +unsigned long delta_ns, +const enum hrtimer_mode mode,int wakeup); +extern int hrtimer_cancel (struct hrtimer *timer);extern int hrtimer_try_to_cancel (struct hrtimer *timer);
include/linux/interrupt.h +1 -0
@@ -274,6 +274,7 @@ extern void softirq_init(void);#define __raise_softirq_irqoff (nr) do { or_softirq_pending(1UL << (nr)); } while (0)extern void raise_softirq_irqoff (unsigned int nr);extern void raise_softirq (unsigned int nr); +extern void wakeup_softirqd (void );/* This is the worklist that queues up per-cpu softirq work. *
kernel/hrtimer.c +34 -21
94 lines not on this page · raw message
kernel/sched.c +11 -3
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)spin_lock (&rt_b->rt_runtime_lock);for (;;) { +unsigned long delta; +ktime_t soft, hard; +if (hrtimer_active (&rt_b->rt_period_timer))break ; now =hrtimer_cb_get_time (&rt_b->rt_period_timer);hrtimer_forward (&rt_b->rt_period_timer, now, rt_b->rt_period); -hrtimer_start_expires (&rt_b->rt_period_timer, -HRTIMER_MODE_ABS ); + + soft =hrtimer_get_softexpires (&rt_b->rt_period_timer); + hard =hrtimer_get_expires (&rt_b->rt_period_timer); + delta =ktime_to_ns (ktime_sub (hard, soft)); +__hrtimer_start_range_ns (&rt_b->rt_period_timer, soft, delta, +HRTIMER_MODE_ABS ,0 ); }spin_unlock (&rt_b->rt_runtime_lock); }
@@ -1129,7 +1136,8 @@ static __init void init_hrtick(void)*/ static void hrtick_start (struct rq *rq,u64 delay) { -hrtimer_start (&rq->hrtick_timer,ns_to_ktime (delay),HRTIMER_MODE_REL ); +__hrtimer_start_range_ns (&rq->hrtick_timer,ns_to_ktime (delay),0 , +HRTIMER_MODE_REL ,0 ); }static inline void init_hrtick (void )
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);* to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static inline void wakeup_softirqd (void ) +void wakeup_softirqd (void ) {/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__get_cpu_var (ksoftirqd);
PATCH 3/11 perf_counter: x86: fix 32bit irq_period assumption.
In reply to Peter ZijlstraNo need to assume the irq_period is 32bit.
--- arch/x86/kernel/cpu/perf_counter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/cpu/perf_counter.c +1 -1
@@ -449,7 +449,7 @@ __hw_perf_counter_set_period(struct perf struct hw_perf_counter *hwc, int idx) {s64 left =atomic64_read (&hwc->period_left); -s32 period = hwc->irq_period; +s64 period = hwc->irq_period;int err;/*
[tip:perfcounters/core] perf_counter: x86: fix 32-bit irq_period assumption
Commit-ID: 83c4b62d58add88f9b3d13062b706d57786dbd46 Gitweb: http://git.kernel.org/tip/83c4b62d58add88f9b3d13062b706d57786dbd46 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:28 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:07 +0100
perf_counter: x86: fix 32-bit irq_period assumption
No need to assume the irq_period is 32bit.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +1 -1
@@ -449,7 +449,7 @@ __hw_perf_counter_set_period(struct perf_counter *counter, struct hw_perf_counter *hwc, int idx) {s64 left =atomic64_read (&hwc->period_left); -s32 period = hwc->irq_period; +s64 period = hwc->irq_period;int err;/*
[tip:perfcounters/core] perf_counter: x86: fix 32-bit irq_period assumption
In reply to Peter ZijlstraCommit-ID: c1c159de20d96a5505dfc0337ca7169a088ebd64 Gitweb: http://git.kernel.org/tip/c1c159de20d96a5505dfc0337ca7169a088ebd64 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:28 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:42 +0100
perf_counter: x86: fix 32-bit irq_period assumption
No need to assume the irq_period is 32bit.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +1 -1
@@ -449,7 +449,7 @@ __hw_perf_counter_set_period(struct perf_counter *counter, struct hw_perf_counter *hwc, int idx) {s64 left =atomic64_read (&hwc->period_left); -s32 period = hwc->irq_period; +s64 period = hwc->irq_period;int err;/*
PATCH 4/11 perf_counter: use list_move_tail
In reply to Peter ZijlstraInstead of del/add use a move list-op.
--- kernel/perf_counter.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
kernel/perf_counter.c +2 -4
@@ -89,8 +89,7 @@ list_del_counter(struct perf_counter *colist_for_each_entry_safe (sibling, tmp, &counter->sibling_list, list_entry) { -list_del_init (&sibling->list_entry); -list_add_tail (&sibling->list_entry, &ctx->counter_list); +list_move_tail (&sibling->list_entry, &ctx->counter_list); sibling->group_leader = sibling; } }
@@ -959,8 +958,7 @@ static void rotate_ctx(struct perf_count*/ perf_flags =hw_perf_save_disable ();list_for_each_entry (counter, &ctx->counter_list, list_entry) { -list_del (&counter->list_entry); -list_add_tail (&counter->list_entry, &ctx->counter_list); +list_move_tail (&counter->list_entry, &ctx->counter_list);break ; }hw_perf_restore (perf_flags);
[tip:perfcounters/core] perf_counter: use list_move_tail()
Commit-ID: 0bc8915410d0819ffb7e6c724df683896e208b46 Gitweb: http://git.kernel.org/tip/0bc8915410d0819ffb7e6c724df683896e208b46 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:29 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:08 +0100
perf_counter: use list_move_tail()
Instead of del/add use a move list-op.
LKML-Reference: <[email protected]>
--- kernel/perf_counter.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-)
kernel/perf_counter.c +2 -4
@@ -89,8 +89,7 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)list_for_each_entry_safe (sibling, tmp, &counter->sibling_list, list_entry) { -list_del_init (&sibling->list_entry); -list_add_tail (&sibling->list_entry, &ctx->counter_list); +list_move_tail (&sibling->list_entry, &ctx->counter_list); sibling->group_leader = sibling; } }
@@ -959,8 +958,7 @@ static void rotate_ctx(struct perf_counter_context *ctx)*/ perf_flags =hw_perf_save_disable ();list_for_each_entry (counter, &ctx->counter_list, list_entry) { -list_del (&counter->list_entry); -list_add_tail (&counter->list_entry, &ctx->counter_list); +list_move_tail (&counter->list_entry, &ctx->counter_list);break ; }hw_perf_restore (perf_flags);
[tip:perfcounters/core] perf_counter: use list_move_tail()
In reply to Peter ZijlstraCommit-ID: 5026af9f8cc04ced259b492a1fa2754c11ddda1a Gitweb: http://git.kernel.org/tip/5026af9f8cc04ced259b492a1fa2754c11ddda1a Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:29 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:43 +0100
perf_counter: use list_move_tail()
Instead of del/add use a move list-op.
LKML-Reference: <[email protected]>
--- kernel/perf_counter.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-)
kernel/perf_counter.c +2 -4
@@ -89,8 +89,7 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)list_for_each_entry_safe (sibling, tmp, &counter->sibling_list, list_entry) { -list_del_init (&sibling->list_entry); -list_add_tail (&sibling->list_entry, &ctx->counter_list); +list_move_tail (&sibling->list_entry, &ctx->counter_list); sibling->group_leader = sibling; } }
@@ -959,8 +958,7 @@ static void rotate_ctx(struct perf_counter_context *ctx)*/ perf_flags =hw_perf_save_disable ();list_for_each_entry (counter, &ctx->counter_list, list_entry) { -list_del (&counter->list_entry); -list_add_tail (&counter->list_entry, &ctx->counter_list); +list_move_tail (&counter->list_entry, &ctx->counter_list);break ; }hw_perf_restore (perf_flags);
PATCH 5/11 perf_counter: add comment to barrier
In reply to Peter ZijlstraWe need to ensure the enabled=0 write happens before we start disabling the actual counters, so that a pcm_amd_enable() will not enable one underneath us.
I think the race is impossible anyway, we always balance the ops within any one context and perform enable() with IRQs disabled.
--- arch/x86/kernel/cpu/perf_counter.c | 4 ++++ 1 file changed, 4 insertions(+)
arch/x86/kernel/cpu/perf_counter.c +4 -0
@@ -247,6 +247,10 @@ static u64 pmc_amd_save_disable_all(void enabled = cpuc->enabled; cpuc->enabled =0 ; +/* +* ensure we write the disable before we start disabling the +* counters proper, so that pcm_amd_enable() does the right thing. +*/ barrier ();for (idx =0 ; idx < nr_counters_generic; idx++) {
[tip:perfcounters/core] perf_counter: add comment to barrier
Commit-ID: 1b1646b16f983a880163afb9efd296de01c7d854 Gitweb: http://git.kernel.org/tip/1b1646b16f983a880163afb9efd296de01c7d854 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:30 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:08 +0100
perf_counter: add comment to barrier
We need to ensure the enabled=0 write happens before we start disabling the actual counters, so that a pcm_amd_enable() will not enable one underneath us.
I think the race is impossible anyway, we always balance the ops within any one context and perform enable() with IRQs disabled.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +4 -0
@@ -247,6 +247,10 @@ static u64 pmc_amd_save_disable_all(void) enabled = cpuc->enabled; cpuc->enabled =0 ; +/* +* ensure we write the disable before we start disabling the +* counters proper, so that pcm_amd_enable() does the right thing. +*/ barrier ();for (idx =0 ; idx < nr_counters_generic; idx++) {
[tip:perfcounters/core] perf_counter: add comment to barrier
In reply to Peter ZijlstraCommit-ID: ffd25c2b454557dd5a09b23d2446d1800d497bde Gitweb: http://git.kernel.org/tip/ffd25c2b454557dd5a09b23d2446d1800d497bde Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:30 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:43 +0100
perf_counter: add comment to barrier
We need to ensure the enabled=0 write happens before we start disabling the actual counters, so that a pcm_amd_enable() will not enable one underneath us.
I think the race is impossible anyway, we always balance the ops within any one context and perform enable() with IRQs disabled.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +4 -0
@@ -247,6 +247,10 @@ static u64 pmc_amd_save_disable_all(void) enabled = cpuc->enabled; cpuc->enabled =0 ; +/* +* ensure we write the disable before we start disabling the +* counters proper, so that pcm_amd_enable() does the right thing. +*/ barrier ();for (idx =0 ; idx < nr_counters_generic; idx++) {
PATCH 6/11 perf_counter: x86: use ULL postfix for 64bit constants
In reply to Peter ZijlstraFix a build warning on 32bit machines by explicitly marking the constants as 64bit.
--- arch/x86/kernel/cpu/perf_counter.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +6 -6
@@ -84,9 +84,9 @@ static u64 pmc_intel_event_map(int eventstatic u64 pmc_intel_raw_event (u64 event) { -#define CORE_EVNTSEL_EVENT_MASK 0x000000FF -#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00 -#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000 +#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL +#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL +#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL#define CORE_EVNTSEL_MASK \ (CORE_EVNTSEL_EVENT_MASK | \
@@ -116,9 +116,9 @@ static u64 pmc_amd_event_map(int event)static u64 pmc_amd_raw_event (u64 event) { -#define K7_EVNTSEL_EVENT_MASK 0x7000000FF -#define K7_EVNTSEL_UNIT_MASK 0x00000FF00 -#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000 +#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL +#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL +#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL#define K7_EVNTSEL_MASK \ (K7_EVNTSEL_EVENT_MASK | \
[tip:perfcounters/core] perf_counter: x86: use ULL postfix for 64bit constants
Commit-ID: c93ccc55a700595a5522a65e5338adcdd76aeeb0 Gitweb: http://git.kernel.org/tip/c93ccc55a700595a5522a65e5338adcdd76aeeb0 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:31 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:09 +0100
perf_counter: x86: use ULL postfix for 64bit constants
Fix a build warning on 32bit machines by explicitly marking the constants as 64-bit.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +6 -6
@@ -84,9 +84,9 @@ static u64 pmc_intel_event_map(int event)static u64 pmc_intel_raw_event (u64 event) { -#define CORE_EVNTSEL_EVENT_MASK 0x000000FF -#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00 -#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000 +#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL +#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL +#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL#define CORE_EVNTSEL_MASK \ (CORE_EVNTSEL_EVENT_MASK | \
@@ -116,9 +116,9 @@ static u64 pmc_amd_event_map(int event)static u64 pmc_amd_raw_event (u64 event) { -#define K7_EVNTSEL_EVENT_MASK 0x7000000FF -#define K7_EVNTSEL_UNIT_MASK 0x00000FF00 -#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000 +#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL +#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL +#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL#define K7_EVNTSEL_MASK \ (K7_EVNTSEL_EVENT_MASK | \
[tip:perfcounters/core] perf_counter: x86: use ULL postfix for 64bit constants
In reply to Peter ZijlstraCommit-ID: 1738302c9b15ec9903f88121b35ff726db304598 Gitweb: http://git.kernel.org/tip/1738302c9b15ec9903f88121b35ff726db304598 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:31 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:43 +0100
perf_counter: x86: use ULL postfix for 64bit constants
Fix a build warning on 32bit machines by explicitly marking the constants as 64-bit.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/cpu/perf_counter.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-)
arch/x86/kernel/cpu/perf_counter.c +6 -6
@@ -84,9 +84,9 @@ static u64 pmc_intel_event_map(int event)static u64 pmc_intel_raw_event (u64 event) { -#define CORE_EVNTSEL_EVENT_MASK 0x000000FF -#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00 -#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000 +#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL +#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL +#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL#define CORE_EVNTSEL_MASK \ (CORE_EVNTSEL_EVENT_MASK | \
@@ -116,9 +116,9 @@ static u64 pmc_amd_event_map(int event)static u64 pmc_amd_raw_event (u64 event) { -#define K7_EVNTSEL_EVENT_MASK 0x7000000FF -#define K7_EVNTSEL_UNIT_MASK 0x00000FF00 -#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000 +#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL +#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL +#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL#define K7_EVNTSEL_MASK \ (K7_EVNTSEL_EVENT_MASK | \
PATCH 7/11 perf_counter: software counter event infrastructure
In reply to Peter ZijlstraProvide generic software counter infrastructure that supports software events.
This will be used to allow sample based profiling based on software events such as pagefaults. The current infrastructure can only provide a count of such events, no place information.
--- include/linux/perf_counter.h | 6 + kernel/perf_counter.c | 201 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+)
include/linux/perf_counter.h +6 -0
@@ -126,6 +126,7 @@ struct hw_perf_counter {unsigned long counter_base;int nmi;unsigned int idx; +atomic64_t count;/* software */ atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
@@ -283,6 +284,8 @@ static inline int is_software_counter(st return !counter->hw_event.raw && counter->hw_event.type < 0; } +extern void perf_swcounter_event (enum hw_event_types ,u64 ,int ,struct pt_regs *); +#else static inline void perf_counter_task_sched_in (struct task_struct *task,int cpu) { }
@@ -299,6 +302,9 @@ static inline void hw_perf_restore(u64 cstatic inline u64 hw_perf_save_disable (void ) {return 0 ; }static inline int perf_counter_task_disable (void ) {return -EINVAL ; }static inline int perf_counter_task_enable (void ) {return -EINVAL ; } + +static inline void perf_swcounter_event (enum hw_event_types event,u64 nr, + int nmi struct pt_regs *regs) { }#endif #endif /* __KERNEL__ */
kernel/perf_counter.c +201 -0
244 lines not on this page · raw message
[tip:perfcounters/core] perf_counter: software counter event infrastructure
Commit-ID: 60b71eab21b8b5d6de1d4746f009ddd0f1568ddb Gitweb: http://git.kernel.org/tip/60b71eab21b8b5d6de1d4746f009ddd0f1568ddb Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:32 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:09 +0100
perf_counter: software counter event infrastructure
Provide generic software counter infrastructure that supports software events.
This will be used to allow sample based profiling based on software events such as pagefaults. The current infrastructure can only provide a count of such events, no place information.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 6 ++ kernel/perf_counter.c | 201 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 0 deletions(-)
include/linux/perf_counter.h +6 -0
@@ -126,6 +126,7 @@ struct hw_perf_counter {unsigned long counter_base;int nmi;unsigned int idx; +atomic64_t count;/* software */ atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
@@ -283,6 +284,8 @@ static inline int is_software_counter(struct perf_counter *counter) return !counter->hw_event.raw && counter->hw_event.type < 0; } +extern void perf_swcounter_event (enum hw_event_types ,u64 ,int ,struct pt_regs *); +#else static inline void perf_counter_task_sched_in (struct task_struct *task,int cpu) { }
@@ -299,6 +302,9 @@ static inline void hw_perf_restore(u64 ctrl) { }static inline u64 hw_perf_save_disable (void ) {return 0 ; }static inline int perf_counter_task_disable (void ) {return -EINVAL ; }static inline int perf_counter_task_enable (void ) {return -EINVAL ; } + +static inline void perf_swcounter_event (enum hw_event_types event,u64 nr, + int nmi struct pt_regs *regs) { }#endif #endif /* __KERNEL__ */
kernel/perf_counter.c +201 -0
244 lines not on this page · raw message
[tip:perfcounters/core] perf_counter: software counter event infrastructure
In reply to Peter ZijlstraCommit-ID: b506f7a61f352b7a1f7b8354ecd35f2c1d526e95 Gitweb: http://git.kernel.org/tip/b506f7a61f352b7a1f7b8354ecd35f2c1d526e95 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:32 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:44 +0100
perf_counter: software counter event infrastructure
Provide generic software counter infrastructure that supports software events.
This will be used to allow sample based profiling based on software events such as pagefaults. The current infrastructure can only provide a count of such events, no place information.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 8 ++- kernel/perf_counter.c | 201 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletions(-)
include/linux/perf_counter.h +7 -1
@@ -126,6 +126,7 @@ struct hw_perf_counter {unsigned long counter_base;int nmi;unsigned int idx; +atomic64_t count;/* software */ atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
@@ -283,6 +284,8 @@ static inline int is_software_counter(struct perf_counter *counter) return !counter->hw_event.raw && counter->hw_event.type < 0; } +extern void perf_swcounter_event (enum hw_event_types ,u64 ,int ,struct pt_regs *); +#else static inline void perf_counter_task_sched_in (struct task_struct *task,int cpu) { }
@@ -295,10 +298,13 @@ static inline void perf_counter_exit_task(struct task_struct *child) { }static inline void perf_counter_notify (struct pt_regs *regs) { }static inline void perf_counter_print_debug (void ) { }static inline void perf_counter_unthrottle (void ) { } -static inline void hw_perf_restore (u64 ctrl) { } +static inline void hw_perf_restore (u64 ctrl) { }static inline u64 hw_perf_save_disable (void ) {return 0 ; }static inline int perf_counter_task_disable (void ) {return -EINVAL ; }static inline int perf_counter_task_enable (void ) {return -EINVAL ; } + +static inline void perf_swcounter_event (enum hw_event_types event,u64 nr, +int nmi,struct pt_regs *regs) { }#endif #endif /* __KERNEL__ */
kernel/perf_counter.c +201 -0
244 lines not on this page · raw message
PATCH 8/11 perf_counter: provide pagefault software events
In reply to Peter ZijlstraWe use the generic software counter infrastructure to provide page fault events.
--- arch/powerpc/mm/fault.c | 3 ++ arch/x86/mm/fault.c | 3 ++ kernel/perf_counter.c | 53 ++---------------------------------------------- 3 files changed, 9 insertions(+), 50 deletions(-)
kernel/perf_counter.c +3 -50
@@ -1601,57 +1601,10 @@ static const struct hw_perf_counter_ops* Software counter: page faults */ -#ifdef CONFIG_VM_EVENT_COUNTERS -#define cpu_page_faults () __get_cpu_var(vm_event_states).event[PGFAULT] -#else -#define cpu_page_faults () 0 -#endif - -static u64 get_page_faults (struct perf_counter *counter) -{ -struct task_struct *curr = counter->ctx->task; - -if (curr) -return curr->maj_flt + curr->min_flt; -return cpu_page_faults (); -} - -static void page_faults_perf_counter_update (struct perf_counter *counter) -{ -u64 prev, now; -s64 delta; - - prev =atomic64_read (&counter->hw.prev_count); - now =get_page_faults (counter); - -atomic64_set (&counter->hw.prev_count, now); - - delta = now - prev; - -atomic64_add (delta, &counter->count); -} - -static void page_faults_perf_counter_read (struct perf_counter *counter) -{ -page_faults_perf_counter_update (counter); -} - -static int page_faults_perf_counter_enable (struct perf_counter *counter) -{ -if (counter->prev_state <=PERF_COUNTER_STATE_OFF ) -atomic64_set (&counter->hw.prev_count,get_page_faults (counter)); -return 0 ; -} - -static void page_faults_perf_counter_disable (struct perf_counter *counter) -{ -page_faults_perf_counter_update (counter); -} -static const struct hw_perf_counter_ops perf_ops_page_faults = { - .enable = page_faults_perf_counter_enable, - .disable = page_faults_perf_counter_disable, - .read = page_faults_perf_counter_read, + .enable = perf_swcounter_enable, + .disable = perf_swcounter_disable, + .read = perf_swcounter_read, };/*
arch/powerpc/mm/fault.c +3 -0
@@ -29,6 +29,7 @@#include <linux/module.h> #include <linux/kprobes.h> #include <linux/kdebug.h> +#include <linux/perf_counter.h> #include <asm/firmware.h> #include <asm/page.h>
@@ -170,6 +171,8 @@ int __kprobes do_page_fault(struct pt_redie ("Weird page fault" , regs,SIGSEGV ); } +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an
arch/x86/mm/fault.c +3 -0
@@ -27,6 +27,7 @@#include <linux/tty.h> #include <linux/smp.h> #include <linux/mm.h> +#include <linux/perf_counter.h> #include <asm-generic/sections.h>
@@ -1056,6 +1057,8 @@ do_page_fault(struct pt_regs *regs, unsiif (unlikely (error_code &PF_RSVD ))pgtable_bad (regs, error_code, address); +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault:
[tip:perfcounters/core] perf_counter: provide pagefault software events
Commit-ID: cfd5e4ffd21a8a46cb6e25f14981c3a53601bbe5 Gitweb: http://git.kernel.org/tip/cfd5e4ffd21a8a46cb6e25f14981c3a53601bbe5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:33 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:10 +0100
perf_counter: provide pagefault software events
We use the generic software counter infrastructure to provide page fault events.
LKML-Reference: <[email protected]>
--- arch/powerpc/mm/fault.c | 3 ++ arch/x86/mm/fault.c | 3 ++ kernel/perf_counter.c | 53 ++-------------------------------------------- 3 files changed, 9 insertions(+), 50 deletions(-)
arch/powerpc/mm/fault.c +3 -0
@@ -29,6 +29,7 @@#include <linux/module.h> #include <linux/kprobes.h> #include <linux/kdebug.h> +#include <linux/perf_counter.h> #include <asm/firmware.h> #include <asm/page.h>
@@ -170,6 +171,8 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,die ("Weird page fault" , regs,SIGSEGV ); } +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an
arch/x86/mm/fault.c +3 -0
@@ -27,6 +27,7 @@#include <linux/tty.h> #include <linux/smp.h> #include <linux/mm.h> +#include <linux/perf_counter.h> #include <asm-generic/sections.h>
@@ -1044,6 +1045,8 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)if (unlikely (error_code &PF_RSVD ))pgtable_bad (regs, error_code, address); +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault:
kernel/perf_counter.c +3 -50
60 lines not on this page · raw message
[tip:perfcounters/core] perf_counter: provide pagefault software events
In reply to Peter ZijlstraCommit-ID: f26827045743f2e12c885980da54306534a87a67 Gitweb: http://git.kernel.org/tip/f26827045743f2e12c885980da54306534a87a67 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:33 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:45 +0100
perf_counter: provide pagefault software events
We use the generic software counter infrastructure to provide page fault events.
LKML-Reference: <[email protected]>
--- arch/powerpc/mm/fault.c | 3 ++ arch/x86/mm/fault.c | 3 ++ kernel/perf_counter.c | 53 ++-------------------------------------------- 3 files changed, 9 insertions(+), 50 deletions(-)
arch/powerpc/mm/fault.c +3 -0
@@ -29,6 +29,7 @@#include <linux/module.h> #include <linux/kprobes.h> #include <linux/kdebug.h> +#include <linux/perf_counter.h> #include <asm/firmware.h> #include <asm/page.h>
@@ -170,6 +171,8 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,die ("Weird page fault" , regs,SIGSEGV ); } +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an
arch/x86/mm/fault.c +3 -0
@@ -27,6 +27,7 @@#include <linux/tty.h> #include <linux/smp.h> #include <linux/mm.h> +#include <linux/perf_counter.h> #include <asm-generic/sections.h>
@@ -1044,6 +1045,8 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)if (unlikely (error_code &PF_RSVD ))pgtable_bad (regs, error_code, address); +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS ,1 ,0 , regs); +/* * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault:
kernel/perf_counter.c +3 -50
60 lines not on this page · raw message
PATCH 9/11 perf_counter: provide major/minor page fault software events
In reply to Peter ZijlstraProvide separate sw counters for major and minor page faults.
--- arch/powerpc/mm/fault.c | 5 ++++- arch/x86/mm/fault.c | 7 +++++-- include/linux/perf_counter.h | 4 +++- kernel/perf_counter.c | 22 +++++++++------------- 4 files changed, 21 insertions(+), 17 deletions(-)
arch/powerpc/mm/fault.c +4 -1
@@ -324,6 +324,7 @@ good_area: }if (ret &VM_FAULT_MAJOR ) { current->maj_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs);#ifdef CONFIG_PPC_SMLPAR if (firmware_has_feature (FW_FEATURE_CMO )) {preempt_disable ();
@@ -331,8 +332,10 @@ good_area: preempt_enable(); }#endif - }else + }else { current->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }up_read (&mm->mmap_sem);return 0 ;
arch/x86/mm/fault.c +5 -2
@@ -1152,10 +1152,13 @@ good_area:return ; } -if (fault &VM_FAULT_MAJOR ) +if (fault &VM_FAULT_MAJOR ) { tsk->maj_flt++; -else +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs); + }else { tsk->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }check_v8086_mode (regs, address, tsk);
include/linux/perf_counter.h +3 -1
@@ -49,8 +49,10 @@ enum hw_event_types {PERF_COUNT_PAGE_FAULTS =-3 ,PERF_COUNT_CONTEXT_SWITCHES =-4 ,PERF_COUNT_CPU_MIGRATIONS =-5 , +PERF_COUNT_PAGE_FAULTS_MIN =-6 , +PERF_COUNT_PAGE_FAULTS_MAJ =-7 , - PERF_SW_EVENTS_MIN = -6, + PERF_SW_EVENTS_MIN = -8, };/*
kernel/perf_counter.c +9 -13
@@ -1513,6 +1513,12 @@ static void perf_swcounter_disable(strucperf_swcounter_update (counter); } +static const struct hw_perf_counter_ops perf_ops_generic = { + .enable = perf_swcounter_enable, + .disable = perf_swcounter_disable, + .read = perf_swcounter_read, +}; +/* * Software counter: cpu wall time clock */
@@ -1614,16 +1620,6 @@ static const struct hw_perf_counter_ops };/* -* Software counter: page faults -*/ - -static const struct hw_perf_counter_ops perf_ops_page_faults = { - .enable = perf_swcounter_enable, - .disable = perf_swcounter_disable, - .read = perf_swcounter_read, -}; - -/* * Software counter: context switches */
@@ -1763,9 +1759,9 @@ sw_perf_counter_init(struct perf_counter hw_ops = &perf_ops_cpu_clock;break ;case PERF_COUNT_PAGE_FAULTS : -if (!(counter->hw_event.exclude_user || - counter->hw_event.exclude_kernel)) - hw_ops = &perf_ops_page_faults; +case PERF_COUNT_PAGE_FAULTS_MIN : +case PERF_COUNT_PAGE_FAULTS_MAJ : + hw_ops = &perf_ops_generic;break ;case PERF_COUNT_CONTEXT_SWITCHES :if (!counter->hw_event.exclude_kernel)
[tip:perfcounters/core] perf_counter: provide major/minor page fault software events
Commit-ID: abeecd0d151291c8056461d8dca7cccaf9a14dcd Gitweb: http://git.kernel.org/tip/abeecd0d151291c8056461d8dca7cccaf9a14dcd Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:34 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:11 +0100
perf_counter: provide major/minor page fault software events
Provide separate sw counters for major and minor page faults.
LKML-Reference: <[email protected]>
--- arch/powerpc/mm/fault.c | 5 ++++- arch/x86/mm/fault.c | 7 +++++-- include/linux/perf_counter.h | 4 +++- kernel/perf_counter.c | 22 +++++++++------------- 4 files changed, 21 insertions(+), 17 deletions(-)
arch/powerpc/mm/fault.c +4 -1
@@ -324,6 +324,7 @@ good_area: }if (ret &VM_FAULT_MAJOR ) { current->maj_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs);#ifdef CONFIG_PPC_SMLPAR if (firmware_has_feature (FW_FEATURE_CMO )) {preempt_disable ();
@@ -331,8 +332,10 @@ good_area: preempt_enable(); }#endif - }else + }else { current->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }up_read (&mm->mmap_sem);return 0 ;
arch/x86/mm/fault.c +5 -2
@@ -1140,10 +1140,13 @@ good_area:return ; } -if (fault &VM_FAULT_MAJOR ) +if (fault &VM_FAULT_MAJOR ) { tsk->maj_flt++; -else +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs); + }else { tsk->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }check_v8086_mode (regs, address, tsk);
include/linux/perf_counter.h +3 -1
@@ -49,8 +49,10 @@ enum hw_event_types {PERF_COUNT_PAGE_FAULTS =-3 ,PERF_COUNT_CONTEXT_SWITCHES =-4 ,PERF_COUNT_CPU_MIGRATIONS =-5 , +PERF_COUNT_PAGE_FAULTS_MIN =-6 , +PERF_COUNT_PAGE_FAULTS_MAJ =-7 , - PERF_SW_EVENTS_MIN = -6, + PERF_SW_EVENTS_MIN = -8, };/*
kernel/perf_counter.c +9 -13
@@ -1503,6 +1503,12 @@ static void perf_swcounter_disable(struct perf_counter *counter)perf_swcounter_update (counter); } +static const struct hw_perf_counter_ops perf_ops_generic = { + .enable = perf_swcounter_enable, + .disable = perf_swcounter_disable, + .read = perf_swcounter_read, +}; +/* * Software counter: cpu wall time clock */
@@ -1604,16 +1610,6 @@ static const struct hw_perf_counter_ops perf_ops_task_clock = { };/* -* Software counter: page faults -*/ - -static const struct hw_perf_counter_ops perf_ops_page_faults = { - .enable = perf_swcounter_enable, - .disable = perf_swcounter_disable, - .read = perf_swcounter_read, -}; - -/* * Software counter: context switches */
@@ -1753,9 +1749,9 @@ sw_perf_counter_init(struct perf_counter *counter) hw_ops = &perf_ops_cpu_clock;break ;case PERF_COUNT_PAGE_FAULTS : -if (!(counter->hw_event.exclude_user || - counter->hw_event.exclude_kernel)) - hw_ops = &perf_ops_page_faults; +case PERF_COUNT_PAGE_FAULTS_MIN : +case PERF_COUNT_PAGE_FAULTS_MAJ : + hw_ops = &perf_ops_generic;break ;case PERF_COUNT_CONTEXT_SWITCHES :if (!counter->hw_event.exclude_kernel)
[tip:perfcounters/core] perf_counter: provide major/minor page fault software events
In reply to Peter ZijlstraCommit-ID: 07904cb8292bad6c020a1a3de499a58969da1624 Gitweb: http://git.kernel.org/tip/07904cb8292bad6c020a1a3de499a58969da1624 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:34 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:45 +0100
perf_counter: provide major/minor page fault software events
Provide separate sw counters for major and minor page faults.
LKML-Reference: <[email protected]>
--- arch/powerpc/mm/fault.c | 5 ++++- arch/x86/mm/fault.c | 7 +++++-- include/linux/perf_counter.h | 4 +++- kernel/perf_counter.c | 22 +++++++++------------- 4 files changed, 21 insertions(+), 17 deletions(-)
arch/powerpc/mm/fault.c +4 -1
@@ -324,6 +324,7 @@ good_area: }if (ret &VM_FAULT_MAJOR ) { current->maj_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs);#ifdef CONFIG_PPC_SMLPAR if (firmware_has_feature (FW_FEATURE_CMO )) {preempt_disable ();
@@ -331,8 +332,10 @@ good_area: preempt_enable(); }#endif - }else + }else { current->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }up_read (&mm->mmap_sem);return 0 ;
arch/x86/mm/fault.c +5 -2
@@ -1140,10 +1140,13 @@ good_area:return ; } -if (fault &VM_FAULT_MAJOR ) +if (fault &VM_FAULT_MAJOR ) { tsk->maj_flt++; -else +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MAJ ,1 ,0 , regs); + }else { tsk->min_flt++; +perf_swcounter_event (PERF_COUNT_PAGE_FAULTS_MIN ,1 ,0 , regs); + }check_v8086_mode (regs, address, tsk);
include/linux/perf_counter.h +3 -1
@@ -49,8 +49,10 @@ enum hw_event_types {PERF_COUNT_PAGE_FAULTS =-3 ,PERF_COUNT_CONTEXT_SWITCHES =-4 ,PERF_COUNT_CPU_MIGRATIONS =-5 , +PERF_COUNT_PAGE_FAULTS_MIN =-6 , +PERF_COUNT_PAGE_FAULTS_MAJ =-7 , - PERF_SW_EVENTS_MIN = -6, + PERF_SW_EVENTS_MIN = -8, };/*
kernel/perf_counter.c +9 -13
@@ -1503,6 +1503,12 @@ static void perf_swcounter_disable(struct perf_counter *counter)perf_swcounter_update (counter); } +static const struct hw_perf_counter_ops perf_ops_generic = { + .enable = perf_swcounter_enable, + .disable = perf_swcounter_disable, + .read = perf_swcounter_read, +}; +/* * Software counter: cpu wall time clock */
@@ -1604,16 +1610,6 @@ static const struct hw_perf_counter_ops perf_ops_task_clock = { };/* -* Software counter: page faults -*/ - -static const struct hw_perf_counter_ops perf_ops_page_faults = { - .enable = perf_swcounter_enable, - .disable = perf_swcounter_disable, - .read = perf_swcounter_read, -}; - -/* * Software counter: context switches */
@@ -1753,9 +1749,9 @@ sw_perf_counter_init(struct perf_counter *counter) hw_ops = &perf_ops_cpu_clock;break ;case PERF_COUNT_PAGE_FAULTS : -if (!(counter->hw_event.exclude_user || - counter->hw_event.exclude_kernel)) - hw_ops = &perf_ops_page_faults; +case PERF_COUNT_PAGE_FAULTS_MIN : +case PERF_COUNT_PAGE_FAULTS_MAJ : + hw_ops = &perf_ops_generic;break ;case PERF_COUNT_CONTEXT_SWITCHES :if (!counter->hw_event.exclude_kernel)
PATCH 10/11 perf_counter: hrtimer based sampling for software time events
In reply to Peter ZijlstraUse hrtimers to profile timer based sampling for the software time counters.
This allows platforms without hardware counter support to still perform sample based profiling.
--- include/linux/perf_counter.h | 20 ++++-- kernel/perf_counter.c | 123 ++++++++++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 43 deletions(-)
kernel/perf_counter.c +86 -37
@@ -1395,7 +1395,7 @@ static void perf_swcounter_handle_group(struct perf_counter *counter, *group_leader = sibling->group_leader;list_for_each_entry (counter, &group_leader->sibling_list, list_entry) { -perf_swcounter_update (counter); + counter->hw_ops->read (counter);perf_swcounter_store_irq (sibling, counter->hw_event.type);perf_swcounter_store_irq (sibling,atomic64_read (&counter->count)); }
@@ -1404,8 +1404,6 @@ static void perf_swcounter_handle_group(static void perf_swcounter_interrupt (struct perf_counter *counter,int nmi,struct pt_regs *regs) { -perf_swcounter_save_and_restart (counter); -switch (counter->hw_event.record_type) {case PERF_RECORD_SIMPLE :break ;
@@ -1426,6 +1424,38 @@ static void perf_swcounter_interrupt(strwake_up (&counter->waitq); } +static enum hrtimer_restart perf_swcounter_hrtimer (struct hrtimer *hrtimer) +{ +struct perf_counter *counter; +struct pt_regs *regs; + + counter = container_of(hrtimer, struct perf_counter, hw.hrtimer); + counter->hw_ops->read (counter); + + regs =get_irq_regs (); +/* +* In case we exclude kernel IPs or are somehow not in interrupt +* context, provide the next best thing, the user IP. +*/ +if ((counter->hw_event.exclude_kernel || !regs) && + !counter->hw_event.exclude_user) + regs =task_pt_regs (current); + +if (regs) +perf_swcounter_interrupt (counter,0 , regs); + +hrtimer_forward_now (hrtimer,ns_to_ktime (counter->hw.irq_period)); + +return HRTIMER_RESTART ; +} + +static void perf_swcounter_overflow (struct perf_counter *counter, +int nmi,struct pt_regs *regs) +{ +perf_swcounter_save_and_restart (counter); +perf_swcounter_interrupt (counter, nmi, regs); +} +static int perf_swcounter_match (struct perf_counter *counter,enum hw_event_types event,struct pt_regs *regs)
@@ -1448,13 +1478,20 @@ static int perf_swcounter_match(struct preturn 1 ; } +static void perf_swcounter_add (struct perf_counter *counter,u64 nr, +int nmi,struct pt_regs *regs) +{ +int neg =atomic64_add_negative (nr, &counter->hw.count); +if (counter->hw.irq_period && !neg) +perf_swcounter_overflow (counter, nmi, regs); +} +static void perf_swcounter_ctx_event (struct perf_counter_context *ctx,enum hw_event_types event,u64 nr,int nmi,struct pt_regs *regs) {struct perf_counter *counter;unsigned long flags; -int neg;if (list_empty (&ctx->counter_list))return ;
@@ -1465,11 +1502,8 @@ static void perf_swcounter_ctx_event(str* XXX: make counter_list RCU safe */ list_for_each_entry (counter, &ctx->counter_list, list_entry) { -if (perf_swcounter_match (counter, event, regs)) { - neg =atomic64_add_negative (nr, &counter->hw.count); -if (counter->hw.irq_period && !neg) -perf_swcounter_interrupt (counter, nmi, regs); - } +if (perf_swcounter_match (counter, event, regs)) +perf_swcounter_add (counter, nr, nmi, regs); }spin_unlock_irqrestore (&ctx->lock, flags);
@@ -1513,14 +1547,6 @@ static const struct hw_perf_counter_ops* Software counter: cpu wall time clock */ -static int cpu_clock_perf_counter_enable (struct perf_counter *counter) -{ -int cpu =raw_smp_processor_id (); - -atomic64_set (&counter->hw.prev_count,cpu_clock (cpu)); -return 0 ; -} -static void cpu_clock_perf_counter_update (struct perf_counter *counter) {int cpu =raw_smp_processor_id ();
@@ -1533,8 +1559,26 @@ static void cpu_clock_perf_counter_updatatomic64_add (now - prev, &counter->count); } +static int cpu_clock_perf_counter_enable (struct perf_counter *counter) +{ +struct hw_perf_counter *hwc = &counter->hw; +int cpu =raw_smp_processor_id (); + +atomic64_set (&hwc->prev_count,cpu_clock (cpu)); +if (hwc->irq_period) { +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer; +__hrtimer_start_range_ns (&hwc->hrtimer, +ns_to_ktime (hwc->irq_period),0 , +HRTIMER_MODE_REL ,0 ); + } + +return 0 ; +} +static void cpu_clock_perf_counter_disable (struct perf_counter *counter) { +hrtimer_cancel (&counter->hw.hrtimer);cpu_clock_perf_counter_update (counter); }
@@ -1580,27 +1624,33 @@ static void task_clock_perf_counter_updaatomic64_add (delta, &counter->count); } -static void task_clock_perf_counter_read (struct perf_counter *counter) -{ -u64 now =task_clock_perf_counter_val (counter,1 ); - -task_clock_perf_counter_update (counter, now); -} -static int task_clock_perf_counter_enable (struct perf_counter *counter) { -if (counter->prev_state <=PERF_COUNTER_STATE_OFF ) -atomic64_set (&counter->hw.prev_count, -task_clock_perf_counter_val (counter,0 )); +struct hw_perf_counter *hwc = &counter->hw; + +atomic64_set (&hwc->prev_count,task_clock_perf_counter_val (counter,0 )); +if (hwc->irq_period) { +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer; +__hrtimer_start_range_ns (&hwc->hrtimer, +ns_to_ktime (hwc->irq_period),0 , +HRTIMER_MODE_REL ,0 ); + }return 0 ; }static void task_clock_perf_counter_disable (struct perf_counter *counter) { -u64 now =task_clock_perf_counter_val (counter,0 ); +hrtimer_cancel (&counter->hw.hrtimer); +task_clock_perf_counter_update (counter, +task_clock_perf_counter_val (counter,0 )); +} -task_clock_perf_counter_update (counter, now); +static void task_clock_perf_counter_read (struct perf_counter *counter) +{ +task_clock_perf_counter_update (counter, +task_clock_perf_counter_val (counter,1 )); }static const struct hw_perf_counter_ops perf_ops_task_clock = {
@@ -1729,16 +1779,12 @@ sw_perf_counter_init(struct perf_counter*/ switch (counter->hw_event.type) {case PERF_COUNT_CPU_CLOCK : -if (!(counter->hw_event.exclude_user || - counter->hw_event.exclude_kernel || - counter->hw_event.exclude_hv)) - hw_ops = &perf_ops_cpu_clock; + hw_ops = &perf_ops_cpu_clock; + +if (hw_event->irq_period && hw_event->irq_period <10000 ) + hw_event->irq_period =10000 ;break ;case PERF_COUNT_TASK_CLOCK : -if (counter->hw_event.exclude_user || - counter->hw_event.exclude_kernel || - counter->hw_event.exclude_hv) -break ;/* * If the user instantiates this as a per-cpu counter, * use the cpu_clock counter instead.
@@ -1747,6 +1793,9 @@ sw_perf_counter_init(struct perf_counter hw_ops = &perf_ops_task_clock; else hw_ops = &perf_ops_cpu_clock; + +if (hw_event->irq_period && hw_event->irq_period <10000 ) + hw_event->irq_period =10000 ;break ;case PERF_COUNT_PAGE_FAULTS :case PERF_COUNT_PAGE_FAULTS_MIN :
include/linux/perf_counter.h +14 -6
@@ -114,6 +114,7 @@ struct perf_counter_hw_event {#include <linux/rculist.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> +#include <linux/hrtimer.h> #include <asm/atomic.h> struct task_struct ;
@@ -123,12 +124,19 @@ struct task_struct;*/ struct hw_perf_counter {#ifdef CONFIG_PERF_COUNTERS -u64 config; -unsigned long config_base; -unsigned long counter_base; -int nmi; -unsigned int idx; -atomic64_t count;/* software */ +union { +struct {/* hardware */ +u64 config; +unsigned long config_base; +unsigned long counter_base; +int nmi; +unsigned int idx; + }; +union {/* software */ +atomic64_t count; +struct hrtimer hrtimer; + }; + };atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
[tip:perfcounters/core] perf_counter: hrtimer based sampling for software time events
Commit-ID: 7484b45d2db110fe69c2cbb09c785a0a19767604 Gitweb: http://git.kernel.org/tip/7484b45d2db110fe69c2cbb09c785a0a19767604 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:35 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:11 +0100
perf_counter: hrtimer based sampling for software time events
Use hrtimers to profile timer based sampling for the software time counters.
This allows platforms without hardware counter support to still perform sample based profiling.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 20 +++++-- kernel/perf_counter.c | 123 +++++++++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 43 deletions(-)
include/linux/perf_counter.h +14 -6
@@ -114,6 +114,7 @@ struct perf_counter_hw_event {#include <linux/rculist.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> +#include <linux/hrtimer.h> #include <asm/atomic.h> struct task_struct ;
@@ -123,12 +124,19 @@ struct task_struct;*/ struct hw_perf_counter {#ifdef CONFIG_PERF_COUNTERS -u64 config; -unsigned long config_base; -unsigned long counter_base; -int nmi; -unsigned int idx; -atomic64_t count;/* software */ +union { +struct {/* hardware */ +u64 config; +unsigned long config_base; +unsigned long counter_base; +int nmi; +unsigned int idx; + }; +union {/* software */ +atomic64_t count; +struct hrtimer hrtimer; + }; + };atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
kernel/perf_counter.c +86 -37
202 lines not on this page · raw message
[tip:perfcounters/core] perf_counter: hrtimer based sampling for software time events
In reply to Peter ZijlstraCommit-ID: 4657834f0b97521139e88cf17befccbf416f500e Gitweb: http://git.kernel.org/tip/4657834f0b97521139e88cf17befccbf416f500e Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:35 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:46 +0100
perf_counter: hrtimer based sampling for software time events
Use hrtimers to profile timer based sampling for the software time counters.
This allows platforms without hardware counter support to still perform sample based profiling.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 20 +++++-- kernel/perf_counter.c | 123 +++++++++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 43 deletions(-)
include/linux/perf_counter.h +14 -6
@@ -114,6 +114,7 @@ struct perf_counter_hw_event {#include <linux/rculist.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> +#include <linux/hrtimer.h> #include <asm/atomic.h> struct task_struct ;
@@ -123,12 +124,19 @@ struct task_struct;*/ struct hw_perf_counter {#ifdef CONFIG_PERF_COUNTERS -u64 config; -unsigned long config_base; -unsigned long counter_base; -int nmi; -unsigned int idx; -atomic64_t count;/* software */ +union { +struct {/* hardware */ +u64 config; +unsigned long config_base; +unsigned long counter_base; +int nmi; +unsigned int idx; + }; +union {/* software */ +atomic64_t count; +struct hrtimer hrtimer; + }; + };atomic64_t prev_count;u64 irq_period;atomic64_t period_left;
kernel/perf_counter.c +86 -37
202 lines not on this page · raw message
PATCH 1/1 perf_counter: fix hrtimer sampling
In reply to Peter ZijlstraFix for the perfstat fubar..
We cannot unconditionally call hrtimer_cancel() without ever having done hrtimer_init() on the thing.
---
kernel/perf_counter.c +4 -4
@@ -1565,9 +1565,9 @@ static int cpu_clock_perf_counter_enableint cpu =raw_smp_processor_id ();atomic64_set (&hwc->prev_count,cpu_clock (cpu)); +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer;if (hwc->irq_period) { -hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); - hwc->hrtimer.function = perf_swcounter_hrtimer;__hrtimer_start_range_ns (&hwc->hrtimer,ns_to_ktime (hwc->irq_period),0 ,HRTIMER_MODE_REL ,0 );
@@ -1629,9 +1629,9 @@ static int task_clock_perf_counter_enablstruct hw_perf_counter *hwc = &counter->hw;atomic64_set (&hwc->prev_count,task_clock_perf_counter_val (counter,0 )); +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer;if (hwc->irq_period) { -hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); - hwc->hrtimer.function = perf_swcounter_hrtimer;__hrtimer_start_range_ns (&hwc->hrtimer,ns_to_ktime (hwc->irq_period),0 ,HRTIMER_MODE_REL ,0 );
[tip:perfcounters/core] perf_counter: fix hrtimer sampling
Commit-ID: a27cdc586523eec54717c8c56ff079b3869770d2 Gitweb: http://git.kernel.org/tip/a27cdc586523eec54717c8c56ff079b3869770d2 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 16:43:47 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 16:59:00 +0100
perf_counter: fix hrtimer sampling
Impact: fix deadlock with perfstat
Fix for the perfstat fubar..
We cannot unconditionally call hrtimer_cancel() without ever having done hrtimer_init() on the thing.
LKML-Reference: <1236959027.22447.149.camel@twins>
--- kernel/perf_counter.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
kernel/perf_counter.c +4 -4
@@ -1571,9 +1571,9 @@ static int cpu_clock_perf_counter_enable(struct perf_counter *counter)int cpu =raw_smp_processor_id ();atomic64_set (&hwc->prev_count,cpu_clock (cpu)); +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer;if (hwc->irq_period) { -hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); - hwc->hrtimer.function = perf_swcounter_hrtimer;__hrtimer_start_range_ns (&hwc->hrtimer,ns_to_ktime (hwc->irq_period),0 ,HRTIMER_MODE_REL ,0 );
@@ -1635,9 +1635,9 @@ static int task_clock_perf_counter_enable(struct perf_counter *counter)struct hw_perf_counter *hwc = &counter->hw;atomic64_set (&hwc->prev_count,task_clock_perf_counter_val (counter,0 )); +hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); + hwc->hrtimer.function = perf_swcounter_hrtimer;if (hwc->irq_period) { -hrtimer_init (&hwc->hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); - hwc->hrtimer.function = perf_swcounter_hrtimer;__hrtimer_start_range_ns (&hwc->hrtimer,ns_to_ktime (hwc->irq_period),0 ,HRTIMER_MODE_REL ,0 );
PATCH 11/11 perf_counter: add an event_list
In reply to Peter ZijlstraI noticed that the counter_list only includes top-level counters, thus perf_swcounter_event() will miss sw-counters in groups.
Since perf_swcounter_event() also wants an RCU safe list, create a new event_list that includes all counters and uses RCU list ops and use call_rcu to free the counter structure.
--- include/linux/perf_counter.h | 4 ++++ kernel/perf_counter.c | 30 +++++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-)
include/linux/perf_counter.h +4 -0
@@ -187,6 +187,7 @@ struct file;struct perf_counter {#ifdef CONFIG_PERF_COUNTERS struct list_head list_entry; +struct list_head event_entry;struct list_head sibling_list;struct perf_counter *group_leader;const struct hw_perf_counter_ops *hw_ops;
@@ -220,6 +221,8 @@ struct perf_counter {struct perf_data *irqdata;struct perf_data *usrdata;struct perf_data data[2 ]; + +struct rcu_head rcu_head;#endif };
@@ -243,6 +246,7 @@ struct perf_counter_context {struct mutex mutex;struct list_head counter_list; +struct list_head event_list;int nr_counters;int nr_active;int is_active;
kernel/perf_counter.c +19 -11
@@ -22,6 +22,7 @@#include <linux/perf_counter.h> #include <linux/mm.h> #include <linux/vmstat.h> +#include <linux/rculist.h> /* * Each CPU has a list of per CPU counters:
@@ -72,6 +73,8 @@ list_add_counter(struct perf_counter *colist_add_tail (&counter->list_entry, &ctx->counter_list); else list_add_tail(&counter->list_entry, &group_leader->sibling_list); + +list_add_rcu (&counter->event_entry, &ctx->event_list); } static void
@@ -80,6 +83,7 @@ list_del_counter(struct perf_counter *costruct perf_counter *sibling, *tmp;list_del_init (&counter->list_entry); +list_del_rcu (&counter->event_entry);/* * If this was a group counter with sibling counters then
@@ -1134,6 +1138,14 @@ static struct perf_counter_context *findreturn ctx; } +static void free_counter_rcu (struct rcu_head *head) +{ +struct perf_counter *counter; + + counter = container_of(head, struct perf_counter, rcu_head); +kfree (counter); +} +/* * Called when the last reference to the file is gone. */
@@ -1152,7 +1164,7 @@ static int perf_release(struct inode *inmutex_unlock (&counter->mutex);mutex_unlock (&ctx->mutex); -kfree (counter); +call_rcu (&counter->rcu_head, free_counter_rcu);put_context (ctx);return 0 ;
@@ -1490,22 +1502,16 @@ static void perf_swcounter_ctx_event(str int nmi, struct pt_regs *regs) { struct perf_counter *counter; -unsigned long flags; -if (list_empty (&ctx->counter_list)) +if (list_empty (&ctx->event_list))return ; -spin_lock_irqsave (&ctx->lock, flags); - -/* -* XXX: make counter_list RCU safe -*/ -list_for_each_entry (counter, &ctx->counter_list, list_entry) { +rcu_read_lock (); +list_for_each_entry_rcu (counter, &ctx->event_list, event_entry) {if (perf_swcounter_match (counter, event, regs))perf_swcounter_add (counter, nr, nmi, regs); } - -spin_unlock_irqrestore (&ctx->lock, flags); +rcu_read_unlock (); } void perf_swcounter_event(enum hw_event_types event, u64 nr,
@@ -1843,6 +1849,7 @@ perf_counter_alloc(struct perf_counter_hmutex_init (&counter->mutex);INIT_LIST_HEAD (&counter->list_entry); +INIT_LIST_HEAD (&counter->event_entry);INIT_LIST_HEAD (&counter->sibling_list);init_waitqueue_head (&counter->waitq);
@@ -1989,6 +1996,7 @@ __perf_counter_init_context(struct perf_spin_lock_init (&ctx->lock);mutex_init (&ctx->mutex);INIT_LIST_HEAD (&ctx->counter_list); +INIT_LIST_HEAD (&ctx->event_list); ctx->task = task; }
[tip:perfcounters/core] perf_counter: add an event_list
Commit-ID: 354f35a16d0ba6d9df80090b8b483b7cd8a1a1a3 Gitweb: http://git.kernel.org/tip/354f35a16d0ba6d9df80090b8b483b7cd8a1a1a3 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:36 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 13:59:12 +0100
perf_counter: add an event_list
I noticed that the counter_list only includes top-level counters, thus perf_swcounter_event() will miss sw-counters in groups.
Since perf_swcounter_event() also wants an RCU safe list, create a new event_list that includes all counters and uses RCU list ops and use call_rcu to free the counter structure.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 4 ++++ kernel/perf_counter.c | 30 +++++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-)
include/linux/perf_counter.h +4 -0
@@ -187,6 +187,7 @@ struct file;struct perf_counter {#ifdef CONFIG_PERF_COUNTERS struct list_head list_entry; +struct list_head event_entry;struct list_head sibling_list;struct perf_counter *group_leader;const struct hw_perf_counter_ops *hw_ops;
@@ -220,6 +221,8 @@ struct perf_counter {struct perf_data *irqdata;struct perf_data *usrdata;struct perf_data data[2 ]; + +struct rcu_head rcu_head;#endif };
@@ -243,6 +246,7 @@ struct perf_counter_context {struct mutex mutex;struct list_head counter_list; +struct list_head event_list;int nr_counters;int nr_active;int is_active;
kernel/perf_counter.c +19 -11
@@ -22,6 +22,7 @@#include <linux/perf_counter.h> #include <linux/mm.h> #include <linux/vmstat.h> +#include <linux/rculist.h> /* * Each CPU has a list of per CPU counters:
@@ -72,6 +73,8 @@ list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)list_add_tail (&counter->list_entry, &ctx->counter_list); else list_add_tail(&counter->list_entry, &group_leader->sibling_list); + +list_add_rcu (&counter->event_entry, &ctx->event_list); } static void
@@ -80,6 +83,7 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)struct perf_counter *sibling, *tmp;list_del_init (&counter->list_entry); +list_del_rcu (&counter->event_entry);/* * If this was a group counter with sibling counters then
@@ -1133,6 +1137,14 @@ static struct perf_counter_context *find_get_context(pid_t pid, int cpu)return ctx; } +static void free_counter_rcu (struct rcu_head *head) +{ +struct perf_counter *counter; + + counter = container_of(head, struct perf_counter, rcu_head); +kfree (counter); +} +/* * Called when the last reference to the file is gone. */
@@ -1151,7 +1163,7 @@ static int perf_release(struct inode *inode, struct file *file)mutex_unlock (&counter->mutex);mutex_unlock (&ctx->mutex); -kfree (counter); +call_rcu (&counter->rcu_head, free_counter_rcu);put_context (ctx);return 0 ;
@@ -1491,22 +1503,16 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, int nmi, struct pt_regs *regs) { struct perf_counter *counter; -unsigned long flags; -if (list_empty (&ctx->counter_list)) +if (list_empty (&ctx->event_list))return ; -spin_lock_irqsave (&ctx->lock, flags); - -/* -* XXX: make counter_list RCU safe -*/ -list_for_each_entry (counter, &ctx->counter_list, list_entry) { +rcu_read_lock (); +list_for_each_entry_rcu (counter, &ctx->event_list, event_entry) {if (perf_swcounter_match (counter, event, regs))perf_swcounter_add (counter, nr, nmi, regs); } - -spin_unlock_irqrestore (&ctx->lock, flags); +rcu_read_unlock (); } void perf_swcounter_event(enum hw_event_types event, u64 nr,
@@ -1846,6 +1852,7 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,mutex_init (&counter->mutex);INIT_LIST_HEAD (&counter->list_entry); +INIT_LIST_HEAD (&counter->event_entry);INIT_LIST_HEAD (&counter->sibling_list);init_waitqueue_head (&counter->waitq);
@@ -1992,6 +1999,7 @@ __perf_counter_init_context(struct perf_counter_context *ctx,spin_lock_init (&ctx->lock);mutex_init (&ctx->mutex);INIT_LIST_HEAD (&ctx->counter_list); +INIT_LIST_HEAD (&ctx->event_list); ctx->task = task; }
[tip:perfcounters/core] perf_counter: add an event_list
In reply to Peter ZijlstraCommit-ID: 93c988346b8ea871d90317e6740003e3144348d4 Gitweb: http://git.kernel.org/tip/93c988346b8ea871d90317e6740003e3144348d4 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 13 Mar 2009 12:21:36 +0100 Commit: Ingo Molnar <[email protected]> CommitDate: Fri, 13 Mar 2009 14:02:46 +0100
perf_counter: add an event_list
I noticed that the counter_list only includes top-level counters, thus perf_swcounter_event() will miss sw-counters in groups.
Since perf_swcounter_event() also wants an RCU safe list, create a new event_list that includes all counters and uses RCU list ops and use call_rcu to free the counter structure.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 4 ++++ kernel/perf_counter.c | 30 +++++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-)
include/linux/perf_counter.h +4 -0
@@ -187,6 +187,7 @@ struct file;struct perf_counter {#ifdef CONFIG_PERF_COUNTERS struct list_head list_entry; +struct list_head event_entry;struct list_head sibling_list;struct perf_counter *group_leader;const struct hw_perf_counter_ops *hw_ops;
@@ -220,6 +221,8 @@ struct perf_counter {struct perf_data *irqdata;struct perf_data *usrdata;struct perf_data data[2 ]; + +struct rcu_head rcu_head;#endif };
@@ -243,6 +246,7 @@ struct perf_counter_context {struct mutex mutex;struct list_head counter_list; +struct list_head event_list;int nr_counters;int nr_active;int is_active;
kernel/perf_counter.c +19 -11
@@ -22,6 +22,7 @@#include <linux/perf_counter.h> #include <linux/mm.h> #include <linux/vmstat.h> +#include <linux/rculist.h> /* * Each CPU has a list of per CPU counters:
@@ -72,6 +73,8 @@ list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)list_add_tail (&counter->list_entry, &ctx->counter_list); else list_add_tail(&counter->list_entry, &group_leader->sibling_list); + +list_add_rcu (&counter->event_entry, &ctx->event_list); } static void
@@ -80,6 +83,7 @@ list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)struct perf_counter *sibling, *tmp;list_del_init (&counter->list_entry); +list_del_rcu (&counter->event_entry);/* * If this was a group counter with sibling counters then
@@ -1133,6 +1137,14 @@ static struct perf_counter_context *find_get_context(pid_t pid, int cpu)return ctx; } +static void free_counter_rcu (struct rcu_head *head) +{ +struct perf_counter *counter; + + counter = container_of(head, struct perf_counter, rcu_head); +kfree (counter); +} +/* * Called when the last reference to the file is gone. */
@@ -1151,7 +1163,7 @@ static int perf_release(struct inode *inode, struct file *file)mutex_unlock (&counter->mutex);mutex_unlock (&ctx->mutex); -kfree (counter); +call_rcu (&counter->rcu_head, free_counter_rcu);put_context (ctx);return 0 ;
@@ -1491,22 +1503,16 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, int nmi, struct pt_regs *regs) { struct perf_counter *counter; -unsigned long flags; -if (list_empty (&ctx->counter_list)) +if (list_empty (&ctx->event_list))return ; -spin_lock_irqsave (&ctx->lock, flags); - -/* -* XXX: make counter_list RCU safe -*/ -list_for_each_entry (counter, &ctx->counter_list, list_entry) { +rcu_read_lock (); +list_for_each_entry_rcu (counter, &ctx->event_list, event_entry) {if (perf_swcounter_match (counter, event, regs))perf_swcounter_add (counter, nr, nmi, regs); } - -spin_unlock_irqrestore (&ctx->lock, flags); +rcu_read_unlock (); } void perf_swcounter_event(enum hw_event_types event, u64 nr,
@@ -1846,6 +1852,7 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,mutex_init (&counter->mutex);INIT_LIST_HEAD (&counter->list_entry); +INIT_LIST_HEAD (&counter->event_entry);INIT_LIST_HEAD (&counter->sibling_list);init_waitqueue_head (&counter->waitq);
@@ -1992,6 +1999,7 @@ __perf_counter_init_context(struct perf_counter_context *ctx,spin_lock_init (&ctx->lock);mutex_init (&ctx->mutex);INIT_LIST_HEAD (&ctx->counter_list); +INIT_LIST_HEAD (&ctx->event_list); ctx->task = task; }
Quoted 11 lines · Peter Zijlstra
This patch set improves the current software counters by providing sampling events for most of them.
The clock based software counters use hrtimers to generate events, allowing sample based profiling without hardware support -- obviously this has limited functionality for kernal based profiling since we disable interrupts quite frequently, but should be good for userspace.
Other events like pagefaults insert simple hooks into the actual pagefault code to generate these events.
Very nice series! Applied to tip:core/locking and
tip:perfcounters/core - thanks Peter!
Fixed a trivial !PERF_COUNTERS build failure as well.
Quoted 2 lines
Future work (in-progress) will hook into the tracepoint infrastructure to to provide profiling events.
Cool!
Especially for pagefaults we have a number of callbacks there:
kmmio, kprobes, kmemcheck, tracing, perfcounters - they could indeed all be consolidated step by step onto a tracepoint.
Ingo