perf_counter: syscall ABI cleanup and mmap() interface
Subsystems (4)
cover perf_counter: syscall ABI cleanup and mmap() interface
tested on x86_64, compile tested for ppc64.
PATCH 1/7 perf_counter: remove the event config bitfields
Since the bitfields turned into a bit of a mess, remove them and rely on good old masks.
--- arch/powerpc/kernel/perf_counter.c | 6 +-- arch/x86/kernel/cpu/perf_counter.c | 8 ++-- include/linux/perf_counter.h | 74 +++++++++++++++++++++++++------------ kernel/perf_counter.c | 22 ++++++----- 4 files changed, 70 insertions(+), 40 deletions(-)
arch/powerpc/kernel/perf_counter.c +3 -3
@@ -602,13 +602,13 @@ hw_perf_counter_init(struct perf_counterreturn NULL ;if ((s64 )counter->hw_event.irq_period <0 )return NULL ; -if (!counter->hw_event.raw_type) { - ev = counter->hw_event.event_id; +if (!perf_event_raw (&counter->hw_event)) { + ev =perf_event_id (&counter->hw_event);if (ev >= ppmu->n_generic || ppmu->generic_events[ev] ==0 )return NULL ; ev = ppmu->generic_events[ev]; }else { - ev = counter->hw_event.raw_event_id; + ev =perf_event_config (&counter->hw_event); } counter->hw.config_base = ev; counter->hw.idx =0 ;
arch/x86/kernel/cpu/perf_counter.c +4 -4
@@ -217,15 +217,15 @@ static int __hw_perf_counter_init(struct/* * Raw event type provide the config in the event structure */ -if (hw_event->raw_type) { - hwc->config |= pmc_ops->raw_event (hw_event->raw_event_id); +if (perf_event_raw (hw_event)) { + hwc->config |= pmc_ops->raw_event (perf_event_config (hw_event)); }else { -if (hw_event->event_id >= pmc_ops->max_events) +if (perf_event_id (hw_event) >= pmc_ops->max_events)return -EINVAL ;/* * The generic map: */ - hwc->config |= pmc_ops->event_map (hw_event->event_id); + hwc->config |= pmc_ops->event_map (perf_event_id (hw_event)); } counter->wakeup_pending =0 ;
include/linux/perf_counter.h +50 -24
96 lines not on this page · raw message
kernel/perf_counter.c +13 -9
@@ -1379,7 +1379,7 @@ static void perf_counter_handle_group(stlist_for_each_entry (sub, &leader->sibling_list, list_entry) {if (sub != counter) sub->hw_ops->read (sub); -perf_counter_store_irq (counter, sub->hw_event.event_config); +perf_counter_store_irq (counter, sub->hw_event.config);perf_counter_store_irq (counter,atomic64_read (&sub->count)); } }
@@ -1489,13 +1489,13 @@ static int perf_swcounter_match(struct pif (counter->state !=PERF_COUNTER_STATE_ACTIVE )return 0 ; -if (counter->hw_event.raw_type) +if (perf_event_raw (&counter->hw_event))return 0 ; -if (counter->hw_event.type != type) +if (perf_event_type (&counter->hw_event) != type)return 0 ; -if (counter->hw_event.event_id != event) +if (perf_event_id (&counter->hw_event) != event)return 0 ;if (counter->hw_event.exclude_user &&user_mode (regs))
@@ -1757,13 +1757,13 @@ extern void ftrace_profile_disable(int);static void tp_perf_counter_destroy (struct perf_counter *counter) { -ftrace_profile_disable (counter->hw_event.event_id); +ftrace_profile_disable (perf_event_id (&counter->hw_event)); }static const struct hw_perf_counter_ops *tp_perf_counter_init (struct perf_counter *counter) { -int event_id = counter->hw_event.event_id; +int event_id =perf_event_id (&counter->hw_event);int ret; ret =ftrace_profile_enable (event_id);
@@ -1797,7 +1797,7 @@ sw_perf_counter_init(struct perf_counter* to be kernel events, and page faults are never hypervisor * events. */ -switch (counter->hw_event.event_id) { +switch (perf_event_id (&counter->hw_event)) {case PERF_COUNT_CPU_CLOCK : hw_ops = &perf_ops_cpu_clock;
@@ -1882,9 +1882,12 @@ perf_counter_alloc(struct perf_counter_h hw_ops =NULL ; -if (hw_event->raw_type) +if (perf_event_raw (hw_event)) { hw_ops =hw_perf_counter_init (counter); -else switch (hw_event->type) { + goto done; + } + +switch (perf_event_type (hw_event)) {case PERF_TYPE_HARDWARE : hw_ops =hw_perf_counter_init (counter);break ;
@@ -1902,6 +1905,7 @@ perf_counter_alloc(struct perf_counter_hkfree (counter);return NULL ; } +done: counter->hw_ops = hw_ops;return counter;
[tip:perfcounters/core] perf_counter: remove the event config bitfields
Commit-ID: 47211edfdf473ff304450dfaa2c949994a19f698 Gitweb: http://git.kernel.org/tip/47211edfdf473ff304450dfaa2c949994a19f698 Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:06 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:08 +0100
perf_counter: remove the event config bitfields
Since the bitfields turned into a bit of a mess, remove them and rely on good old masks.
LKML-Reference: <[email protected]>
--- arch/powerpc/kernel/perf_counter.c | 6 +- arch/x86/kernel/cpu/perf_counter.c | 8 ++-- include/linux/perf_counter.h | 74 ++++++++++++++++++++++++------------ kernel/perf_counter.c | 22 ++++++---- 4 files changed, 70 insertions(+), 40 deletions(-)
arch/powerpc/kernel/perf_counter.c +3 -3
@@ -602,13 +602,13 @@ hw_perf_counter_init(struct perf_counter *counter)return NULL ;if ((s64 )counter->hw_event.irq_period <0 )return NULL ; -if (!counter->hw_event.raw_type) { - ev = counter->hw_event.event_id; +if (!perf_event_raw (&counter->hw_event)) { + ev =perf_event_id (&counter->hw_event);if (ev >= ppmu->n_generic || ppmu->generic_events[ev] ==0 )return NULL ; ev = ppmu->generic_events[ev]; }else { - ev = counter->hw_event.raw_event_id; + ev =perf_event_config (&counter->hw_event); } counter->hw.config_base = ev; counter->hw.idx =0 ;
arch/x86/kernel/cpu/perf_counter.c +4 -4
@@ -217,15 +217,15 @@ static int __hw_perf_counter_init(struct perf_counter *counter)/* * Raw event type provide the config in the event structure */ -if (hw_event->raw_type) { - hwc->config |= pmc_ops->raw_event (hw_event->raw_event_id); +if (perf_event_raw (hw_event)) { + hwc->config |= pmc_ops->raw_event (perf_event_config (hw_event)); }else { -if (hw_event->event_id >= pmc_ops->max_events) +if (perf_event_id (hw_event) >= pmc_ops->max_events)return -EINVAL ;/* * The generic map: */ - hwc->config |= pmc_ops->event_map (hw_event->event_id); + hwc->config |= pmc_ops->event_map (perf_event_id (hw_event)); } counter->wakeup_pending =0 ;
include/linux/perf_counter.h +50 -24
96 lines not on this page · raw message
kernel/perf_counter.c +13 -9
@@ -1379,7 +1379,7 @@ static void perf_counter_handle_group(struct perf_counter *counter)list_for_each_entry (sub, &leader->sibling_list, list_entry) {if (sub != counter) sub->hw_ops->read (sub); -perf_counter_store_irq (counter, sub->hw_event.event_config); +perf_counter_store_irq (counter, sub->hw_event.config);perf_counter_store_irq (counter,atomic64_read (&sub->count)); } }
@@ -1489,13 +1489,13 @@ static int perf_swcounter_match(struct perf_counter *counter,if (counter->state !=PERF_COUNTER_STATE_ACTIVE )return 0 ; -if (counter->hw_event.raw_type) +if (perf_event_raw (&counter->hw_event))return 0 ; -if (counter->hw_event.type != type) +if (perf_event_type (&counter->hw_event) != type)return 0 ; -if (counter->hw_event.event_id != event) +if (perf_event_id (&counter->hw_event) != event)return 0 ;if (counter->hw_event.exclude_user &&user_mode (regs))
@@ -1757,13 +1757,13 @@ extern void ftrace_profile_disable(int);static void tp_perf_counter_destroy (struct perf_counter *counter) { -ftrace_profile_disable (counter->hw_event.event_id); +ftrace_profile_disable (perf_event_id (&counter->hw_event)); }static const struct hw_perf_counter_ops *tp_perf_counter_init (struct perf_counter *counter) { -int event_id = counter->hw_event.event_id; +int event_id =perf_event_id (&counter->hw_event);int ret; ret =ftrace_profile_enable (event_id);
@@ -1797,7 +1797,7 @@ sw_perf_counter_init(struct perf_counter *counter)* to be kernel events, and page faults are never hypervisor * events. */ -switch (counter->hw_event.event_id) { +switch (perf_event_id (&counter->hw_event)) {case PERF_COUNT_CPU_CLOCK : hw_ops = &perf_ops_cpu_clock;
@@ -1882,9 +1882,12 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event, hw_ops =NULL ; -if (hw_event->raw_type) +if (perf_event_raw (hw_event)) { hw_ops =hw_perf_counter_init (counter); -else switch (hw_event->type) { + goto done; + } + +switch (perf_event_type (hw_event)) {case PERF_TYPE_HARDWARE : hw_ops =hw_perf_counter_init (counter);break ;
@@ -1902,6 +1905,7 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,kfree (counter);return NULL ; } +done: counter->hw_ops = hw_ops;return counter;
PATCH 2/7 perf_counter: avoid recursion
In reply to Peter ZijlstraTracepoint events like lock_acquire and software counters like pagefaults can recurse into the perf counter code again, avoid that.
--- include/linux/perf_counter.h | 7 +++++++ kernel/perf_counter.c | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+)
include/linux/perf_counter.h +7 -0
@@ -328,6 +328,13 @@ struct perf_cpu_context {int active_oncpu;int max_pertask;int exclusive; + +/* +* Recursion avoidance: +* +* task, softirq, irq, nmi context +*/ +int recursion[4 ]; };/*
kernel/perf_counter.c +26 -0
@@ -23,6 +23,7 @@#include <linux/mm.h> #include <linux/vmstat.h> #include <linux/rculist.h> +#include <linux/hardirq.h> #include <asm/irq_regs.h>
@@ -1532,10 +1533,31 @@ static void perf_swcounter_ctx_event(strrcu_read_unlock (); } +static int *perf_swcounter_recursion_context (struct perf_cpu_context *cpuctx) +{ +if (in_nmi ()) +return &cpuctx->recursion[3 ]; + +if (in_irq ()) +return &cpuctx->recursion[2 ]; + +if (in_softirq ()) +return &cpuctx->recursion[1 ]; + +return &cpuctx->recursion[0 ]; +} +static void __perf_swcounter_event (enum perf_event_types type,u32 event,u64 nr,int nmi,struct pt_regs *regs) {struct perf_cpu_context *cpuctx = &get_cpu_var (perf_cpu_context); +int *recursion =perf_swcounter_recursion_context (cpuctx); + +if (*recursion) + goto out; + + (*recursion)++; +barrier ();perf_swcounter_ctx_event (&cpuctx->ctx, type, event, nr, nmi, regs);if (cpuctx->task_ctx) {
@@ -1543,6 +1565,10 @@ static void __perf_swcounter_event(enum nr, nmi, regs); } +barrier (); + (*recursion)--; + +out:put_cpu_var (perf_cpu_context); }
[tip:perfcounters/core] perf_counter: avoid recursion
Commit-ID: 4d0ec5f64b60057b395f2d314189ccde71d2d0bd Gitweb: http://git.kernel.org/tip/4d0ec5f64b60057b395f2d314189ccde71d2d0bd Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:07 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:09 +0100
perf_counter: avoid recursion
Tracepoint events like lock_acquire and software counters like pagefaults can recurse into the perf counter code again, avoid that.
LKML-Reference: <[email protected]>
--- include/linux/perf_counter.h | 7 +++++++ kernel/perf_counter.c | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-)
include/linux/perf_counter.h +7 -0
@@ -328,6 +328,13 @@ struct perf_cpu_context {int active_oncpu;int max_pertask;int exclusive; + +/* +* Recursion avoidance: +* +* task, softirq, irq, nmi context +*/ +int recursion[4 ]; };/*
kernel/perf_counter.c +26 -0
@@ -23,6 +23,7 @@#include <linux/mm.h> #include <linux/vmstat.h> #include <linux/rculist.h> +#include <linux/hardirq.h> #include <asm/irq_regs.h>
@@ -1532,10 +1533,31 @@ static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,rcu_read_unlock (); } +static int *perf_swcounter_recursion_context (struct perf_cpu_context *cpuctx) +{ +if (in_nmi ()) +return &cpuctx->recursion[3 ]; + +if (in_irq ()) +return &cpuctx->recursion[2 ]; + +if (in_softirq ()) +return &cpuctx->recursion[1 ]; + +return &cpuctx->recursion[0 ]; +} +static void __perf_swcounter_event (enum perf_event_types type,u32 event,u64 nr,int nmi,struct pt_regs *regs) {struct perf_cpu_context *cpuctx = &get_cpu_var (perf_cpu_context); +int *recursion =perf_swcounter_recursion_context (cpuctx); + +if (*recursion) + goto out; + + (*recursion)++; +barrier ();perf_swcounter_ctx_event (&cpuctx->ctx, type, event, nr, nmi, regs);if (cpuctx->task_ctx) {
@@ -1543,6 +1565,10 @@ static void __perf_swcounter_event(enum perf_event_types type, u32 event, nr, nmi, regs); } +barrier (); + (*recursion)--; + +out:put_cpu_var (perf_cpu_context); }
PATCH 3/7 perf_counter: add an mmap method to allow userspace to read hardware counters
In reply to Peter ZijlstraFrom: Paul Mackerras <[email protected]>
Impact: new feature giving performance improvement
This adds the ability for userspace to do an mmap on a hardware counter fd and get access to a read-only page that contains the information needed to translate a hardware counter value to the full 64-bit counter value that would be returned by a read on the fd. This is useful on architectures that allow user programs to read the hardware counters, such as PowerPC.
The mmap will only succeed if the counter is a hardware counter monitoring the current process.
On my quad 2.5GHz PowerPC 970MP machine, userspace can read a counter and translate it to the full 64-bit value in about 30ns using the mmapped page, compared to about 830ns for the read syscall on the counter, so this does give a significant performance improvement.
--- arch/powerpc/kernel/perf_counter.c | 6 ++ include/linux/perf_counter.h | 15 +++++++ kernel/perf_counter.c | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+)
arch/powerpc/kernel/perf_counter.c +6 -0
@@ -417,6 +417,8 @@ void hw_perf_restore(u64 disable)atomic64_set (&counter->hw.prev_count, val); counter->hw.idx = hwc_index[i] +1 ;write_pmc (counter->hw.idx, val); +if (counter->user_page) +perf_counter_update_userpage (counter); }mb (); cpuhw->mmcr[0 ] |=MMCR0_PMXE |MMCR0_FCECE ;
@@ -572,6 +574,8 @@ static void power_perf_disable(struct pe ppmu->disable_pmc (counter->hw.idx -1 , cpuhw->mmcr);write_pmc (counter->hw.idx,0 ); counter->hw.idx =0 ; +if (counter->user_page) +perf_counter_update_userpage (counter);break ; } }
@@ -697,6 +701,8 @@ static void record_and_restart(struct pewrite_pmc (counter->hw.idx, val);atomic64_set (&counter->hw.prev_count, val);atomic64_set (&counter->hw.period_left, left); +if (counter->user_page) +perf_counter_update_userpage (counter);/* * Finally record data if requested.
include/linux/perf_counter.h +15 -0
@@ -126,6 +126,17 @@ struct perf_counter_hw_event {#define PERF_COUNTER_IOC_ENABLE _IO('$', 0)#define PERF_COUNTER_IOC_DISABLE _IO('$', 1) +/* +* Structure of the page that can be mapped via mmap +*/ +struct perf_counter_mmap_page { +__u32 version;/* version number of this structure */ +__u32 compat_version;/* lowest version this is compat with */ +__u32 lock;/* seqlock for synchronization */ +__u32 index;/* hardware counter identifier */ +__s64 offset;/* add to hardware counter value */ +}; +#ifdef __KERNEL__/* * Kernel-internal data types and definitions:
@@ -240,6 +251,9 @@ struct perf_counter {int oncpu;int cpu; +/* pointer to page shared with userspace via mmap */ +unsigned long user_page; +/* read() / irq related data */ wait_queue_head_t waitq;/* optional: for NMIs */
@@ -316,6 +330,7 @@ extern int perf_counter_task_enable(voidextern int hw_perf_group_sched_in (struct perf_counter *group_leader,struct perf_cpu_context *cpuctx,struct perf_counter_context *ctx,int cpu); +extern void perf_counter_update_userpage (struct perf_counter *counter);extern void perf_counter_output (struct perf_counter *counter,int nmi,struct pt_regs *regs);
kernel/perf_counter.c +76 -0
94 lines not on this page · raw message
[tip:perfcounters/core] perf_counter: add an mmap method to allow userspace to read hardware counters
Commit-ID: 36e6cd42be5579128495e7d9e678638f4945de6e Gitweb: http://git.kernel.org/tip/36e6cd42be5579128495e7d9e678638f4945de6e Author: Paul Mackerras <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:08 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:09 +0100
perf_counter: add an mmap method to allow userspace to read hardware counters
Impact: new feature giving performance improvement
This adds the ability for userspace to do an mmap on a hardware counter fd and get access to a read-only page that contains the information needed to translate a hardware counter value to the full 64-bit counter value that would be returned by a read on the fd. This is useful on architectures that allow user programs to read the hardware counters, such as PowerPC.
The mmap will only succeed if the counter is a hardware counter monitoring the current process.
On my quad 2.5GHz PowerPC 970MP machine, userspace can read a counter and translate it to the full 64-bit value in about 30ns using the mmapped page, compared to about 830ns for the read syscall on the counter, so this does give a significant performance improvement.
LKML-Reference: <[email protected]>
--- arch/powerpc/kernel/perf_counter.c | 6 +++ include/linux/perf_counter.h | 15 +++++++ kernel/perf_counter.c | 76 ++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 0 deletions(-)
arch/powerpc/kernel/perf_counter.c +6 -0
@@ -417,6 +417,8 @@ void hw_perf_restore(u64 disable)atomic64_set (&counter->hw.prev_count, val); counter->hw.idx = hwc_index[i] +1 ;write_pmc (counter->hw.idx, val); +if (counter->user_page) +perf_counter_update_userpage (counter); }mb (); cpuhw->mmcr[0 ] |=MMCR0_PMXE |MMCR0_FCECE ;
@@ -572,6 +574,8 @@ static void power_perf_disable(struct perf_counter *counter) ppmu->disable_pmc (counter->hw.idx -1 , cpuhw->mmcr);write_pmc (counter->hw.idx,0 ); counter->hw.idx =0 ; +if (counter->user_page) +perf_counter_update_userpage (counter);break ; } }
@@ -698,6 +702,8 @@ static void record_and_restart(struct perf_counter *counter, long val,write_pmc (counter->hw.idx, val);atomic64_set (&counter->hw.prev_count, val);atomic64_set (&counter->hw.period_left, left); +if (counter->user_page) +perf_counter_update_userpage (counter);/* * Finally record data if requested.
include/linux/perf_counter.h +15 -0
@@ -143,6 +143,17 @@ struct perf_counter_hw_event {#define PERF_COUNTER_IOC_ENABLE _IO('$', 0)#define PERF_COUNTER_IOC_DISABLE _IO('$', 1) +/* +* Structure of the page that can be mapped via mmap +*/ +struct perf_counter_mmap_page { +__u32 version;/* version number of this structure */ +__u32 compat_version;/* lowest version this is compat with */ +__u32 lock;/* seqlock for synchronization */ +__u32 index;/* hardware counter identifier */ +__s64 offset;/* add to hardware counter value */ +}; +#ifdef __KERNEL__/* * Kernel-internal data types and definitions:
@@ -278,6 +289,9 @@ struct perf_counter {int oncpu;int cpu; +/* pointer to page shared with userspace via mmap */ +unsigned long user_page; +/* read() / irq related data */ wait_queue_head_t waitq;/* optional: for NMIs */
@@ -361,6 +375,7 @@ extern int perf_counter_task_enable(void);extern int hw_perf_group_sched_in (struct perf_counter *group_leader,struct perf_cpu_context *cpuctx,struct perf_counter_context *ctx,int cpu); +extern void perf_counter_update_userpage (struct perf_counter *counter);extern void perf_counter_output (struct perf_counter *counter,int nmi,struct pt_regs *regs);
kernel/perf_counter.c +76 -0
94 lines not on this page · raw message
PATCH 4/7 mutex: add atomic_dec_and_mutex_lock]
In reply to Peter ZijlstraFrom: Eric Paris <[email protected]>
Much like the atomic_dec_and_lock() function in which we take an hold a spin_lock if we drop the atomic to 0 this function takes and holds the mutex if we dec the atomic to 0.
--- include/linux/mutex.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
include/linux/mutex.h +23 -0
@@ -151,4 +151,27 @@ extern int __must_check mutex_lock_killaextern int mutex_trylock (struct mutex *lock);extern void mutex_unlock (struct mutex *lock); +/** +* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 +* @cnt: the atomic which we are to dec +* @lock: the mutex to return holding if we dec to 0 +* +* return true and hold lock if we dec to 0, return false otherwise +*/ +static inline int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock) +{ +/* dec if we can't possibly hit 0 */ +if (atomic_add_unless (cnt,-1 ,1 )) +return 0 ; +/* we might hit 0, so take the lock */ +mutex_lock (lock); +if (!atomic_dec_and_test (cnt)) { +/* when we actually did the dec, we didn't hit 0 */ +mutex_unlock (lock); +return 0 ; + } +/* we hit 0, and we hold the lock */ +return 1 ; +} +#endif
[tip:perfcounters/core] mutex: add atomic_dec_and_mutex_lock()
Commit-ID: 3a2d367d9aabac486ac4444c6c7ec7a1dab16267 Gitweb: http://git.kernel.org/tip/3a2d367d9aabac486ac4444c6c7ec7a1dab16267 Author: Eric Paris <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:09 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:10 +0100
mutex: add atomic_dec_and_mutex_lock()
Much like the atomic_dec_and_lock() function in which we take an hold a spin_lock if we drop the atomic to 0 this function takes and holds the mutex if we dec the atomic to 0.
LKML-Reference: <[email protected]>
--- include/linux/mutex.h | 23 +++++++++++++++++++++++ 1 files changed, 23 insertions(+), 0 deletions(-)
include/linux/mutex.h +23 -0
@@ -151,4 +151,27 @@ extern int __must_check mutex_lock_killable(struct mutex *lock);extern int mutex_trylock (struct mutex *lock);extern void mutex_unlock (struct mutex *lock); +/** +* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 +* @cnt: the atomic which we are to dec +* @lock: the mutex to return holding if we dec to 0 +* +* return true and hold lock if we dec to 0, return false otherwise +*/ +static inline int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock) +{ +/* dec if we can't possibly hit 0 */ +if (atomic_add_unless (cnt,-1 ,1 )) +return 0 ; +/* we might hit 0, so take the lock */ +mutex_lock (lock); +if (!atomic_dec_and_test (cnt)) { +/* when we actually did the dec, we didn't hit 0 */ +mutex_unlock (lock); +return 0 ; + } +/* we hit 0, and we hold the lock */ +return 1 ; +} +#endif
[tip:perfcounters/core] mutex: drop "inline" from mutex_lock() inside kernel/mutex.c
Commit-ID: 655be4a209fbcae85a745027f426e1f9879b5648 Gitweb: http://git.kernel.org/tip/655be4a209fbcae85a745027f426e1f9879b5648 Author: H. Peter Anvin <[email protected]> AuthorDate: Wed, 1 Apr 2009 17:21:56 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Wed, 1 Apr 2009 17:21:56 -0700
mutex: drop "inline" from mutex_lock() inside kernel/mutex.c
Impact: build fix
mutex_lock() is was defined inline in kernel/mutex.c, but wasn't declared so not in <linux/mutex.h>. This didn't cause a problem until checkin 3a2d367d9aabac486ac4444c6c7ec7a1dab16267 added the atomic_dec_and_mutex_lock() inline in between declaration and definion.
This broke building with CONFIG_ALLOW_WARNINGS=n, e.g. make allnoconfig.
Either from the source code nor the allnoconfig binary output I cannot find any internal references to mutex_lock() in kernel/mutex.c, so presumably this "inline" is now-useless legacy.
Cc: Eric Paris <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Paul Mackerras <[email protected]>
LKML-Reference: <[email protected]>
--- kernel/mutex.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
kernel/mutex.c +1 -1
@@ -89,7 +89,7 @@ __mutex_lock_slowpath(atomic_t *lock_count);* * This function is similar to (but not equivalent to) down(). */ -void inline __sched mutex_lock(struct mutex *lock) +void __sched mutex_lock(struct mutex *lock) {might_sleep ();/*
PATCH 5/7 perf_counter: new output ABI - part 1
In reply to Peter ZijlstraRework the output ABI
use sys_read() only for instant data and provide mmap() output for all async overflow data.
The first mmap() determines the size of the output buffer. The mmap() size must be a PAGE_SIZE multiple of 1+pages, where pages must be a power of 2 or 0. Further mmap()s of the same fd must have the same size. Once all maps are gone, you can again mmap() with a new size.
In case of 0 extra pages there is no data output and the first page only contains meta data.
When there are data pages, a poll() event will be generated for each full page of data. Furthermore, the output is circular. This means that although 1 page is a valid configuration, its useless, since we'll start overwriting it the instant we report a full page.
Future work will focus on the output format (currently maintained) where we'll likey want each entry denoted by a header which includes a type and length.
Further future work will allow to splice() the fd, also containing the async overflow data -- splice() would be mutually exclusive with mmap() of the data.
--- arch/powerpc/kernel/perf_counter.c | 9 include/linux/perf_counter.h | 36 +- kernel/perf_counter.c | 464 +++++++++++++++++++------------------ 3 files changed, 263 insertions(+), 246 deletions(-)
include/linux/perf_counter.h +15 -21
@@ -152,6 +152,8 @@ struct perf_counter_mmap_page {__u32 lock;/* seqlock for synchronization */ __u32 index;/* hardware counter identifier */ __s64 offset;/* add to hardware counter value */ + +__u32 data_head;/* head in the data section */ };#ifdef __KERNEL__
@@ -218,21 +220,6 @@ struct hw_perf_counter {#endif }; -/* -* Hardcoded buffer length limit for now, for IRQ-fed events: -*/ -#define PERF_DATA_BUFLEN 2048 - -/** -* struct perf_data - performance counter IRQ data sampling ... -*/ -struct perf_data { -int len; -int rd_idx; -int overrun; -u8 data[PERF_DATA_BUFLEN ]; -}; -struct perf_counter ;/**
@@ -256,6 +243,14 @@ enum perf_counter_active_state {struct file ; +struct perf_mmap_data { +struct rcu_head rcu_head; +int nr_pages; +atomic_t head; +struct perf_counter_mmap_page *user_page; +void *data_pages[0 ]; +}; +/** * struct perf_counter - performance counter kernel representation: */
@@ -289,16 +284,15 @@ struct perf_counter {int oncpu;int cpu; -/* pointer to page shared with userspace via mmap */ -unsigned long user_page; +/* mmap bits */ +struct mutex mmap_mutex; +atomic_t mmap_count; +struct perf_mmap_data *data; -/* read() / irq related data */ +/* poll related */ wait_queue_head_t waitq;/* optional: for NMIs */ int wakeup_pending; -struct perf_data *irqdata; -struct perf_data *usrdata; -struct perf_data data[2 ];void (*destroy)(struct perf_counter *);struct rcu_head rcu_head;
kernel/perf_counter.c +245 -219
590 lines not on this page · raw message
arch/powerpc/kernel/perf_counter.c +3 -6
@@ -417,8 +417,7 @@ void hw_perf_restore(u64 disable)atomic64_set (&counter->hw.prev_count, val); counter->hw.idx = hwc_index[i] +1 ;write_pmc (counter->hw.idx, val); -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter); }mb (); cpuhw->mmcr[0 ] |=MMCR0_PMXE |MMCR0_FCECE ;
@@ -574,8 +573,7 @@ static void power_perf_disable(struct pe ppmu->disable_pmc (counter->hw.idx -1 , cpuhw->mmcr);write_pmc (counter->hw.idx,0 ); counter->hw.idx =0 ; -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter);break ; } }
@@ -702,8 +700,7 @@ static void record_and_restart(struct pewrite_pmc (counter->hw.idx, val);atomic64_set (&counter->hw.prev_count, val);atomic64_set (&counter->hw.period_left, left); -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter);/* * Finally record data if requested.
[tip:perfcounters/core] perf_counter: new output ABI - part 1
Commit-ID: 6329237a583dd1d6ef9a8ebfe87eb59cdde26b2a Gitweb: http://git.kernel.org/tip/6329237a583dd1d6ef9a8ebfe87eb59cdde26b2a Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:10 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:10 +0100
perf_counter: new output ABI - part 1
Impact: Rework the perfcounter output ABI
use sys_read() only for instant data and provide mmap() output for all async overflow data.
The first mmap() determines the size of the output buffer. The mmap() size must be a PAGE_SIZE multiple of 1+pages, where pages must be a power of 2 or 0. Further mmap()s of the same fd must have the same size. Once all maps are gone, you can again mmap() with a new size.
In case of 0 extra pages there is no data output and the first page only contains meta data.
When there are data pages, a poll() event will be generated for each full page of data. Furthermore, the output is circular. This means that although 1 page is a valid configuration, its useless, since we'll start overwriting it the instant we report a full page.
Future work will focus on the output format (currently maintained) where we'll likey want each entry denoted by a header which includes a type and length.
Further future work will allow to splice() the fd, also containing the async overflow data -- splice() would be mutually exclusive with mmap() of the data.
LKML-Reference: <[email protected]>
--- arch/powerpc/kernel/perf_counter.c | 9 +- include/linux/perf_counter.h | 36 ++-- kernel/perf_counter.c | 464 +++++++++++++++++++----------------- 3 files changed, 263 insertions(+), 246 deletions(-)
arch/powerpc/kernel/perf_counter.c +3 -6
@@ -417,8 +417,7 @@ void hw_perf_restore(u64 disable)atomic64_set (&counter->hw.prev_count, val); counter->hw.idx = hwc_index[i] +1 ;write_pmc (counter->hw.idx, val); -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter); }mb (); cpuhw->mmcr[0 ] |=MMCR0_PMXE |MMCR0_FCECE ;
@@ -574,8 +573,7 @@ static void power_perf_disable(struct perf_counter *counter) ppmu->disable_pmc (counter->hw.idx -1 , cpuhw->mmcr);write_pmc (counter->hw.idx,0 ); counter->hw.idx =0 ; -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter);break ; } }
@@ -702,8 +700,7 @@ static void record_and_restart(struct perf_counter *counter, long val,write_pmc (counter->hw.idx, val);atomic64_set (&counter->hw.prev_count, val);atomic64_set (&counter->hw.period_left, left); -if (counter->user_page) -perf_counter_update_userpage (counter); +perf_counter_update_userpage (counter);/* * Finally record data if requested.
include/linux/perf_counter.h +15 -21
@@ -152,6 +152,8 @@ struct perf_counter_mmap_page {__u32 lock;/* seqlock for synchronization */ __u32 index;/* hardware counter identifier */ __s64 offset;/* add to hardware counter value */ + +__u32 data_head;/* head in the data section */ };#ifdef __KERNEL__
@@ -218,21 +220,6 @@ struct hw_perf_counter {#endif }; -/* -* Hardcoded buffer length limit for now, for IRQ-fed events: -*/ -#define PERF_DATA_BUFLEN 2048 - -/** -* struct perf_data - performance counter IRQ data sampling ... -*/ -struct perf_data { -int len; -int rd_idx; -int overrun; -u8 data[PERF_DATA_BUFLEN ]; -}; -struct perf_counter ;/**
@@ -256,6 +243,14 @@ enum perf_counter_active_state {struct file ; +struct perf_mmap_data { +struct rcu_head rcu_head; +int nr_pages; +atomic_t head; +struct perf_counter_mmap_page *user_page; +void *data_pages[0 ]; +}; +/** * struct perf_counter - performance counter kernel representation: */
@@ -289,16 +284,15 @@ struct perf_counter {int oncpu;int cpu; -/* pointer to page shared with userspace via mmap */ -unsigned long user_page; +/* mmap bits */ +struct mutex mmap_mutex; +atomic_t mmap_count; +struct perf_mmap_data *data; -/* read() / irq related data */ +/* poll related */ wait_queue_head_t waitq;/* optional: for NMIs */ int wakeup_pending; -struct perf_data *irqdata; -struct perf_data *usrdata; -struct perf_data data[2 ];void (*destroy)(struct perf_counter *);struct rcu_head rcu_head;
kernel/perf_counter.c +245 -219
590 lines not on this page · raw message
PATCH 6/7 kerneltop: update to new syscall ABI
In reply to Peter Zijlstraupdate the kerneltop userspace to work with the latest syscall ABI
--- kerneltop.c | 235 ++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 157 insertions(+), 78 deletions(-)
kerneltop.c +157 -78
@@ -87,20 +87,90 @@#include <linux/unistd.h> -#include "perfcounters.h" +#include "include/linux/perf_counter.h" +/* +* prctl(PR_TASK_PERF_COUNTERS_DISABLE) will (cheaply) disable all +* counters in the current task. +*/ +#define PR_TASK_PERF_COUNTERS_DISABLE 31 +#define PR_TASK_PERF_COUNTERS_ENABLE 32 + +#define ARRAY_SIZE (x) (sizeof(x) / sizeof((x)[0])) + +#define rdclock () \ +({ \ + struct timespec ts; \ + \ + clock_gettime(CLOCK_MONOTONIC, &ts); \ + ts.tv_sec * 1000000000ULL + ts.tv_nsec; \ +}) + +/* +* Pick up some kernel type conventions: +*/ +#define __user +#define asmlinkage + +typedef unsigned int __u32 ; +typedef unsigned long long __u64 ; +typedef long long __s64 ; + + +#ifdef __x86_64__ +# define __NR_perf_counter_open 295 +#endif + +#ifdef __i386__ +# define __NR_perf_counter_open 333 +#endif + +#ifdef __powerpc__ +#define __NR_perf_counter_open 319 +#endif + +asmlinkage int sys_perf_counter_open( +struct perf_counter_hw_event *hw_event_uptr__user , +pid_t pid, +int cpu, +int group_fd, +unsigned long flags) +{ +int ret; + + ret =syscall ( + __NR_perf_counter_open, hw_event_uptr, pid, cpu, group_fd, flags); +#if defined(__x86_64__) || defined(__i386__) +if (ret <0 && ret >-4096 ) { + errno = -ret; + ret =-1 ; + } +#endif +return ret; +} +#define MAX_COUNTERS 64#define MAX_NR_CPUS 256 -#define DEF_PERFSTAT_EVENTS { -2, -5, -4, -3, 0, 1, 2, 3} +#define EID (type, id) (((__u64)(type) << PERF_COUNTER_TYPE_SHIFT) | (id))static int run_perfstat =0 ;static int system_wide =0 ;static int nr_counters =0 ; -static __s64 event_id[MAX_COUNTERS ] =DEF_PERFSTAT_EVENTS ; -static int event_raw[MAX_COUNTERS ]; +static __u64 event_id[MAX_COUNTERS ] = { +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_TASK_CLOCK ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ), + +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_INSTRUCTIONS ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_REFERENCES ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_MISSES ), +}; +static int default_interval =100000 ;static int event_count[MAX_COUNTERS ];static int fd[MAX_NR_CPUS ][MAX_COUNTERS ];
@@ -156,49 +226,63 @@ static char *sw_event_names[] = {"pagefaults" ,"context switches" ,"CPU migrations" , +"minor faults" , + "major faults", };struct event_symbol { -int event; +__u64 event;char *symbol; };static struct event_symbol event_symbols[] = { - {PERF_COUNT_CPU_CYCLES ,"cpu-cycles" , }, - {PERF_COUNT_CPU_CYCLES ,"cycles" , }, - {PERF_COUNT_INSTRUCTIONS ,"instructions" , }, - {PERF_COUNT_CACHE_REFERENCES ,"cache-references" , }, - {PERF_COUNT_CACHE_MISSES ,"cache-misses" , }, - {PERF_COUNT_BRANCH_INSTRUCTIONS ,"branch-instructions" , }, - {PERF_COUNT_BRANCH_INSTRUCTIONS ,"branches" , }, - {PERF_COUNT_BRANCH_MISSES ,"branch-misses" , }, - {PERF_COUNT_BUS_CYCLES ,"bus-cycles" , }, - {PERF_COUNT_CPU_CLOCK ,"cpu-ticks" , }, - {PERF_COUNT_CPU_CLOCK ,"ticks" , }, - {PERF_COUNT_TASK_CLOCK ,"task-ticks" , }, - {PERF_COUNT_PAGE_FAULTS ,"page-faults" , }, - {PERF_COUNT_PAGE_FAULTS ,"faults" , }, - {PERF_COUNT_CONTEXT_SWITCHES ,"context-switches" , }, - {PERF_COUNT_CONTEXT_SWITCHES ,"cs" , }, - {PERF_COUNT_CPU_MIGRATIONS ,"cpu-migrations" , }, - {PERF_COUNT_CPU_MIGRATIONS ,"migrations" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ),"cpu-cycles" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ),"cycles" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_INSTRUCTIONS ),"instructions" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_REFERENCES ),"cache-references" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_MISSES ),"cache-misses" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_INSTRUCTIONS ),"branch-instructions" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_INSTRUCTIONS ),"branches" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_MISSES ),"branch-misses" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BUS_CYCLES ),"bus-cycles" , }, + + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_CLOCK ),"cpu-clock" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_TASK_CLOCK ),"task-clock" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ),"page-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ),"faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS_MIN ),"minor-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS_MAJ ),"major-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CONTEXT_SWITCHES ),"context-switches" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CONTEXT_SWITCHES ),"cs" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ),"cpu-migrations" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ),"migrations" , }, }; +#define __PERF_COUNTER_FIELD (config, name) \ + ((config & PERF_COUNTER_##name##_MASK) >> PERF_COUNTER_##name##_SHIFT) + +#define PERF_COUNTER_RAW (config) __PERF_COUNTER_FIELD(config, RAW) +#define PERF_COUNTER_CONFIG (config) __PERF_COUNTER_FIELD(config, CONFIG) +#define PERF_COUNTER_TYPE (config) __PERF_COUNTER_FIELD(config, TYPE) +#define PERF_COUNTER_ID (config) __PERF_COUNTER_FIELD(config, EVENT) +static void display_events_help (void ) {unsigned int i; -int e; +__u64 e;printf (" -e EVENT --event=EVENT # symbolic-name abbreviations" ); -for (i =0 , e =PERF_HW_EVENTS_MAX ; i <ARRAY_SIZE (event_symbols); i++) { -if (e != event_symbols[i].event) { - e = event_symbols[i].event; -printf ( -"\n %2d: %-20s" , e, event_symbols[i].symbol); - }else -printf (" %s" , event_symbols[i].symbol); +for (i =0 ; i <ARRAY_SIZE (event_symbols); i++) { +int type, id; + + e = event_symbols[i].event; + type =PERF_COUNTER_TYPE (e); + id =PERF_COUNTER_ID (e); + +printf ("\n %d:%d: %-20s" , + type, id, event_symbols[i].symbol); } printf("\n"
@@ -249,44 +333,51 @@ static void display_help(void) exit(0); } -static int type_valid(int type) -{ - if (type >= PERF_HW_EVENTS_MAX) - return 0; - if (type <= PERF_SW_EVENTS_MIN) - return 0; - - return 1; -} - static char *event_name(int ctr) { - __s64 type = event_id[ctr]; + __u64 config = event_id[ctr]; + int type = PERF_COUNTER_TYPE(config); + int id = PERF_COUNTER_ID(config); static char buf[32]; - if (event_raw[ctr]) { - sprintf(buf, "raw 0x%llx", (long long)type); + if (PERF_COUNTER_RAW(config)) { + sprintf(buf, "raw 0x%llx", PERF_COUNTER_CONFIG(config)); return buf; } - if (!type_valid(type)) - return "unknown"; - if (type >= 0) - return hw_event_names[type]; + switch (type) { + case PERF_TYPE_HARDWARE: + if (id < PERF_HW_EVENTS_MAX) + return hw_event_names[id]; + return "unknown-hardware"; + + case PERF_TYPE_SOFTWARE: + if (id < PERF_SW_EVENTS_MAX) + return sw_event_names[id]; + return "unknown-software"; + + default: + break; + } - return sw_event_names[-type-1]; + return "unknown"; } /* * Each event can have multiple symbolic names. * Symbolic names are (almost) exactly matched. */ -static int match_event_symbols(char *str) +static __u64 match_event_symbols(char *str) { + __u64 config, id; + int type; unsigned int i; - if (isdigit(str[0]) || str[0] == '-') - return atoi(str); + if (sscanf(str, "r%llx", &config) == 1) + return config | PERF_COUNTER_RAW_MASK; + + if (sscanf(str, "%d:%llu", &type, &id) == 2) + return EID(type, id); for (i = 0; i < ARRAY_SIZE(event_symbols); i++) { if (!strncmp(str, event_symbols[i].symbol,
@@ -294,31 +385,22 @@ static int match_event_symbols(char *strreturn event_symbols[i].event; } -return PERF_HW_EVENTS_MAX ; +return ~0ULL ; }static int parse_events (char *str) { -__s64 type; -int raw; +__u64 config; again:if (nr_counters ==MAX_COUNTERS )return -1 ; - raw =0 ; -if (*str =='r' ) { - raw =1 ; - ++str; - type =strtol (str,NULL ,16 ); - }else { - type =match_event_symbols (str); -if (!type_valid (type)) -return -1 ; - } + config =match_event_symbols (str); +if (config == ~0ULL ) +return -1 ; - event_id[nr_counters] = type; - event_raw[nr_counters] = raw; + event_id[nr_counters] = config; nr_counters++; str =strstr (str,"," );
@@ -342,8 +424,7 @@ static void create_perfstat_counter(intstruct perf_counter_hw_event hw_event;memset (&hw_event,0 ,sizeof (hw_event)); - hw_event.type = event_id[counter]; - hw_event.raw = event_raw[counter]; + hw_event.config = event_id[counter]; hw_event.record_type =PERF_RECORD_SIMPLE ; hw_event.nmi =0 ;
@@ -428,7 +509,7 @@ int do_perfstat(int argc, char *argv[]) count += single_count; } -if (!event_raw[counter] && +if (!PERF_COUNTER_RAW (event_id[counter]) && (event_id[counter] ==PERF_COUNT_CPU_CLOCK || event_id[counter] ==PERF_COUNT_TASK_CLOCK )) {
@@ -911,7 +992,7 @@ static void record_ip(uint64_t ip, int cassert (left <= middle && middle <= right);if (!(left <= ip && ip <= right)) {printf (" left: %016lx\n" , left); -printf (" ip: %016lx\n" , ip); +printf (" ip: %016llx\n" , ip);printf ("right: %016lx\n" , right); }assert (left <= ip && ip <= right);
@@ -983,7 +1064,7 @@ static void process_options(int argc, chswitch (c) {case 'a' : system_wide =1 ;break ; -case 'c' : event_count[nr_counters] =atoi (optarg);break ; +case 'c' : default_interval =atoi (optarg);break ;case 'C' :/* CPU and PID are mutually exclusive */ if (tid !=-1 ) {
@@ -1032,10 +1113,7 @@ static void process_options(int argc, chif (event_count[counter])continue ; -if (event_id[counter] <PERF_HW_EVENTS_MAX ) - event_count[counter] = default_count[event_id[counter]]; -else - event_count[counter] =100000 ; + event_count[counter] = default_interval; } }
@@ -1070,12 +1148,13 @@ int main(int argc, char *argv[]) cpu = i;memset (&hw_event,0 ,sizeof (hw_event)); - hw_event.type = event_id[counter]; - hw_event.raw = event_raw[counter]; + hw_event.config = event_id[counter]; hw_event.irq_period = event_count[counter]; hw_event.record_type =PERF_RECORD_IRQ ; hw_event.nmi = nmi; +printf ("FOO: %d %llx %llx\n" , counter, event_id[counter], event_count[counter]); + fd[i][counter] =sys_perf_counter_open (&hw_event, tid, cpu, group_fd,0 );fcntl (fd[i][counter],F_SETFL ,O_NONBLOCK );if (fd[i][counter] <0 ) {
[tip:perfcounters/core] kerneltop: update to new syscall ABI
Commit-ID: d68cd6cac7e4b63ca990e7c4c24aef112f8d6c60 Gitweb: http://git.kernel.org/tip/d68cd6cac7e4b63ca990e7c4c24aef112f8d6c60 Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:11 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:11 +0100
kerneltop: update to new syscall ABI
update the kerneltop userspace to work with the latest syscall ABI
LKML-Reference: <[email protected]>
--- Documentation/perf_counter/kerneltop.c | 235 +++++++++++++++++++++----------- 1 files changed, 157 insertions(+), 78 deletions(-)
Documentation/perf_counter/kerneltop.c +157 -78
@@ -87,20 +87,90 @@#include <linux/unistd.h> -#include "perfcounters.h" +#include "include/linux/perf_counter.h" +/* +* prctl(PR_TASK_PERF_COUNTERS_DISABLE) will (cheaply) disable all +* counters in the current task. +*/ +#define PR_TASK_PERF_COUNTERS_DISABLE 31 +#define PR_TASK_PERF_COUNTERS_ENABLE 32 + +#define ARRAY_SIZE (x) (sizeof(x) / sizeof((x)[0])) + +#define rdclock () \ +({ \ + struct timespec ts; \ + \ + clock_gettime(CLOCK_MONOTONIC, &ts); \ + ts.tv_sec * 1000000000ULL + ts.tv_nsec; \ +}) + +/* +* Pick up some kernel type conventions: +*/ +#define __user +#define asmlinkage + +typedef unsigned int __u32 ; +typedef unsigned long long __u64 ; +typedef long long __s64 ; + + +#ifdef __x86_64__ +# define __NR_perf_counter_open 295 +#endif + +#ifdef __i386__ +# define __NR_perf_counter_open 333 +#endif + +#ifdef __powerpc__ +#define __NR_perf_counter_open 319 +#endif + +asmlinkage int sys_perf_counter_open( +struct perf_counter_hw_event *hw_event_uptr__user , +pid_t pid, +int cpu, +int group_fd, +unsigned long flags) +{ +int ret; + + ret =syscall ( + __NR_perf_counter_open, hw_event_uptr, pid, cpu, group_fd, flags); +#if defined(__x86_64__) || defined(__i386__) +if (ret <0 && ret >-4096 ) { + errno = -ret; + ret =-1 ; + } +#endif +return ret; +} +#define MAX_COUNTERS 64#define MAX_NR_CPUS 256 -#define DEF_PERFSTAT_EVENTS { -2, -5, -4, -3, 0, 1, 2, 3} +#define EID (type, id) (((__u64)(type) << PERF_COUNTER_TYPE_SHIFT) | (id))static int run_perfstat =0 ;static int system_wide =0 ;static int nr_counters =0 ; -static __s64 event_id[MAX_COUNTERS ] =DEF_PERFSTAT_EVENTS ; -static int event_raw[MAX_COUNTERS ]; +static __u64 event_id[MAX_COUNTERS ] = { +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_TASK_CLOCK ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ), +EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ), + +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_INSTRUCTIONS ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_REFERENCES ), +EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_MISSES ), +}; +static int default_interval =100000 ;static int event_count[MAX_COUNTERS ];static int fd[MAX_NR_CPUS ][MAX_COUNTERS ];
@@ -156,49 +226,63 @@ static char *sw_event_names[] = {"pagefaults" ,"context switches" ,"CPU migrations" , +"minor faults" , + "major faults", };struct event_symbol { -int event; +__u64 event;char *symbol; };static struct event_symbol event_symbols[] = { - {PERF_COUNT_CPU_CYCLES ,"cpu-cycles" , }, - {PERF_COUNT_CPU_CYCLES ,"cycles" , }, - {PERF_COUNT_INSTRUCTIONS ,"instructions" , }, - {PERF_COUNT_CACHE_REFERENCES ,"cache-references" , }, - {PERF_COUNT_CACHE_MISSES ,"cache-misses" , }, - {PERF_COUNT_BRANCH_INSTRUCTIONS ,"branch-instructions" , }, - {PERF_COUNT_BRANCH_INSTRUCTIONS ,"branches" , }, - {PERF_COUNT_BRANCH_MISSES ,"branch-misses" , }, - {PERF_COUNT_BUS_CYCLES ,"bus-cycles" , }, - {PERF_COUNT_CPU_CLOCK ,"cpu-ticks" , }, - {PERF_COUNT_CPU_CLOCK ,"ticks" , }, - {PERF_COUNT_TASK_CLOCK ,"task-ticks" , }, - {PERF_COUNT_PAGE_FAULTS ,"page-faults" , }, - {PERF_COUNT_PAGE_FAULTS ,"faults" , }, - {PERF_COUNT_CONTEXT_SWITCHES ,"context-switches" , }, - {PERF_COUNT_CONTEXT_SWITCHES ,"cs" , }, - {PERF_COUNT_CPU_MIGRATIONS ,"cpu-migrations" , }, - {PERF_COUNT_CPU_MIGRATIONS ,"migrations" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ),"cpu-cycles" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CPU_CYCLES ),"cycles" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_INSTRUCTIONS ),"instructions" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_REFERENCES ),"cache-references" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_CACHE_MISSES ),"cache-misses" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_INSTRUCTIONS ),"branch-instructions" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_INSTRUCTIONS ),"branches" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BRANCH_MISSES ),"branch-misses" , }, + {EID (PERF_TYPE_HARDWARE ,PERF_COUNT_BUS_CYCLES ),"bus-cycles" , }, + + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_CLOCK ),"cpu-clock" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_TASK_CLOCK ),"task-clock" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ),"page-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS ),"faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS_MIN ),"minor-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_PAGE_FAULTS_MAJ ),"major-faults" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CONTEXT_SWITCHES ),"context-switches" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CONTEXT_SWITCHES ),"cs" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ),"cpu-migrations" , }, + {EID (PERF_TYPE_SOFTWARE ,PERF_COUNT_CPU_MIGRATIONS ),"migrations" , }, }; +#define __PERF_COUNTER_FIELD (config, name) \ + ((config & PERF_COUNTER_##name##_MASK) >> PERF_COUNTER_##name##_SHIFT) + +#define PERF_COUNTER_RAW (config) __PERF_COUNTER_FIELD(config, RAW) +#define PERF_COUNTER_CONFIG (config) __PERF_COUNTER_FIELD(config, CONFIG) +#define PERF_COUNTER_TYPE (config) __PERF_COUNTER_FIELD(config, TYPE) +#define PERF_COUNTER_ID (config) __PERF_COUNTER_FIELD(config, EVENT) +static void display_events_help (void ) {unsigned int i; -int e; +__u64 e;printf (" -e EVENT --event=EVENT # symbolic-name abbreviations" ); -for (i =0 , e =PERF_HW_EVENTS_MAX ; i <ARRAY_SIZE (event_symbols); i++) { -if (e != event_symbols[i].event) { - e = event_symbols[i].event; -printf ( -"\n %2d: %-20s" , e, event_symbols[i].symbol); - }else -printf (" %s" , event_symbols[i].symbol); +for (i =0 ; i <ARRAY_SIZE (event_symbols); i++) { +int type, id; + + e = event_symbols[i].event; + type =PERF_COUNTER_TYPE (e); + id =PERF_COUNTER_ID (e); + +printf ("\n %d:%d: %-20s" , + type, id, event_symbols[i].symbol); } printf("\n"
@@ -249,44 +333,51 @@ static void display_help(void) exit(0); } -static int type_valid(int type) -{ - if (type >= PERF_HW_EVENTS_MAX) - return 0; - if (type <= PERF_SW_EVENTS_MIN) - return 0; - - return 1; -} - static char *event_name(int ctr) { - __s64 type = event_id[ctr]; + __u64 config = event_id[ctr]; + int type = PERF_COUNTER_TYPE(config); + int id = PERF_COUNTER_ID(config); static char buf[32]; - if (event_raw[ctr]) { - sprintf(buf, "raw 0x%llx", (long long)type); + if (PERF_COUNTER_RAW(config)) { + sprintf(buf, "raw 0x%llx", PERF_COUNTER_CONFIG(config)); return buf; } - if (!type_valid(type)) - return "unknown"; - if (type >= 0) - return hw_event_names[type]; + switch (type) { + case PERF_TYPE_HARDWARE: + if (id < PERF_HW_EVENTS_MAX) + return hw_event_names[id]; + return "unknown-hardware"; + + case PERF_TYPE_SOFTWARE: + if (id < PERF_SW_EVENTS_MAX) + return sw_event_names[id]; + return "unknown-software"; - return sw_event_names[-type-1]; + default: + break; + } + + return "unknown"; } /* * Each event can have multiple symbolic names. * Symbolic names are (almost) exactly matched. */ -static int match_event_symbols(char *str) +static __u64 match_event_symbols(char *str) { + __u64 config, id; + int type; unsigned int i; - if (isdigit(str[0]) || str[0] == '-') - return atoi(str); + if (sscanf(str, "r%llx", &config) == 1) + return config | PERF_COUNTER_RAW_MASK; + + if (sscanf(str, "%d:%llu", &type, &id) == 2) + return EID(type, id); for (i = 0; i < ARRAY_SIZE(event_symbols); i++) { if (!strncmp(str, event_symbols[i].symbol,
@@ -294,31 +385,22 @@ static int match_event_symbols(char *str)return event_symbols[i].event; } -return PERF_HW_EVENTS_MAX ; +return ~0ULL ; }static int parse_events (char *str) { -__s64 type; -int raw; +__u64 config; again:if (nr_counters ==MAX_COUNTERS )return -1 ; - raw =0 ; -if (*str =='r' ) { - raw =1 ; - ++str; - type =strtol (str,NULL ,16 ); - }else { - type =match_event_symbols (str); -if (!type_valid (type)) -return -1 ; - } + config =match_event_symbols (str); +if (config == ~0ULL ) +return -1 ; - event_id[nr_counters] = type; - event_raw[nr_counters] = raw; + event_id[nr_counters] = config; nr_counters++; str =strstr (str,"," );
@@ -342,8 +424,7 @@ static void create_perfstat_counter(int counter)struct perf_counter_hw_event hw_event;memset (&hw_event,0 ,sizeof (hw_event)); - hw_event.type = event_id[counter]; - hw_event.raw = event_raw[counter]; + hw_event.config = event_id[counter]; hw_event.record_type =PERF_RECORD_SIMPLE ; hw_event.nmi =0 ;
@@ -428,7 +509,7 @@ int do_perfstat(int argc, char *argv[]) count += single_count; } -if (!event_raw[counter] && +if (!PERF_COUNTER_RAW (event_id[counter]) && (event_id[counter] ==PERF_COUNT_CPU_CLOCK || event_id[counter] ==PERF_COUNT_TASK_CLOCK )) {
@@ -911,7 +992,7 @@ static void record_ip(uint64_t ip, int counter)assert (left <= middle && middle <= right);if (!(left <= ip && ip <= right)) {printf (" left: %016lx\n" , left); -printf (" ip: %016lx\n" , ip); +printf (" ip: %016llx\n" , ip);printf ("right: %016lx\n" , right); }assert (left <= ip && ip <= right);
@@ -983,7 +1064,7 @@ static void process_options(int argc, char *argv[])switch (c) {case 'a' : system_wide =1 ;break ; -case 'c' : event_count[nr_counters] =atoi (optarg);break ; +case 'c' : default_interval =atoi (optarg);break ;case 'C' :/* CPU and PID are mutually exclusive */ if (tid !=-1 ) {
@@ -1032,10 +1113,7 @@ static void process_options(int argc, char *argv[])if (event_count[counter])continue ; -if (event_id[counter] <PERF_HW_EVENTS_MAX ) - event_count[counter] = default_count[event_id[counter]]; -else - event_count[counter] =100000 ; + event_count[counter] = default_interval; } }
@@ -1070,12 +1148,13 @@ int main(int argc, char *argv[]) cpu = i;memset (&hw_event,0 ,sizeof (hw_event)); - hw_event.type = event_id[counter]; - hw_event.raw = event_raw[counter]; + hw_event.config = event_id[counter]; hw_event.irq_period = event_count[counter]; hw_event.record_type =PERF_RECORD_IRQ ; hw_event.nmi = nmi; +printf ("FOO: %d %llx %llx\n" , counter, event_id[counter], event_count[counter]); + fd[i][counter] =sys_perf_counter_open (&hw_event, tid, cpu, group_fd,0 );fcntl (fd[i][counter],F_SETFL ,O_NONBLOCK );if (fd[i][counter] <0 ) {
PATCH 7/7 kerneltop: use mmap() output
In reply to Peter Zijlstraupdate kerneltop to use the mmap() output to gather overflow information
--- kerneltop.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 14 deletions(-)
kerneltop.c +79 -14
@@ -84,6 +84,7 @@#include <sys/prctl.h> #include <sys/wait.h> #include <sys/uio.h> +#include <sys/mman.h> #include <linux/unistd.h>
@@ -119,17 +120,25 @@ typedef long long __s64;#ifdef __x86_64__ -# define __NR_perf_counter_open 295 +#define __NR_perf_counter_open 295 +#define rmb () asm volatile("lfence" ::: "memory") +#define cpu_relax () asm volatile("rep; nop" ::: "memory");#endif #ifdef __i386__ -# define __NR_perf_counter_open 333 +#define __NR_perf_counter_open 333 +#define rmb () asm volatile("lfence" ::: "memory") +#define cpu_relax () asm volatile("rep; nop" ::: "memory");#endif #ifdef __powerpc__#define __NR_perf_counter_open 319 +#define rmb () asm volatile ("sync" ::: "memory") +#define cpu_relax () asm volatile ("" ::: "memory");#endif +#define unlikely (x) __builtin_expect(!!(x), 0) + asmlinkage int sys_perf_counter_open( struct perf_counter_hw_event *hw_event_uptr __user, pid_t pid,
@@ -181,6 +190,7 @@ static int profile_cpu = -1;static int nr_cpus =0 ;static int nmi =1 ;static int group =0 ; +static unsigned int page_size;static char *vmlinux;
@@ -1117,16 +1127,68 @@ static void process_options(int argc, ch } } +struct mmap_data { +int counter; +void *base; +unsigned int mask; +unsigned int prev; +}; + +static unsigned int mmap_read_head (struct mmap_data *md) +{ +struct perf_counter_mmap_page *pc = md->base; +unsigned int seq, head; + +repeat: +rmb (); + seq = pc->lock; + +if (unlikely (seq &1 )) { +cpu_relax (); + goto repeat; + } + + head = pc->data_head; + +rmb (); +if (pc->lock != seq) + goto repeat; + +return head; +} + +static void mmap_read (struct mmap_data *md) +{ +unsigned int head =mmap_read_head (md); +unsigned int old = md->prev; +unsigned char *data = md->base + page_size; + +if (head - old > md->mask) { +printf ("ERROR: failed to keep up with mmap data\n" ); +exit (-1 ); + } + +for (; old != head;) { +__u64 *ptr = (__u64 *)&data[old & md->mask]; + old +=sizeof (__u64); + +process_event (*ptr, md->counter); + } + + md->prev = old; +} +int main (int argc,char *argv[]) {struct pollfd event_array[MAX_NR_CPUS ][MAX_COUNTERS ]; +struct mmap_data mmap_array[MAX_NR_CPUS ][MAX_COUNTERS ];struct perf_counter_hw_event hw_event;int i, counter, group_fd;unsigned int cpu; -uint64_t ip; -ssize_t res;int ret; + page_size =sysconf (_SC_PAGE_SIZE); +process_options (argc, argv); nr_cpus =sysconf (_SC_NPROCESSORS_ONLN);
@@ -1153,8 +1215,6 @@ int main(int argc, char *argv[]) hw_event.record_type =PERF_RECORD_IRQ ; hw_event.nmi = nmi; -printf ("FOO: %d %llx %llx\n" , counter, event_id[counter], event_count[counter]); - fd[i][counter] =sys_perf_counter_open (&hw_event, tid, cpu, group_fd,0 );fcntl (fd[i][counter],F_SETFL ,O_NONBLOCK );if (fd[i][counter] <0 ) {
@@ -1174,6 +1234,17 @@ int main(int argc, char *argv[]) event_array[i][counter].fd = fd[i][counter]; event_array[i][counter].events =POLLIN ; + + mmap_array[i][counter].counter = counter; + mmap_array[i][counter].prev =0 ; + mmap_array[i][counter].mask =2 *page_size -1 ; + mmap_array[i][counter].base =mmap (NULL ,3 *page_size, +PROT_READ ,MAP_SHARED , fd[i][counter],0 ); +if (mmap_array[i][counter].base ==MAP_FAILED ) { +printf ("kerneltop error: failed to mmap with %d (%s)\n" , + errno,strerror (errno)); +exit (-1 ); + } } }
@@ -1188,14 +1259,8 @@ int main(int argc, char *argv[])int hits = events;for (i =0 ; i < nr_cpus; i++) { -for (counter =0 ; counter < nr_counters; counter++) { - res =read (fd[i][counter], (char *) &ip,sizeof (ip)); -if (res >0 ) { -assert (res ==sizeof (ip)); - -process_event (ip, counter); - } - } +for (counter =0 ; counter < nr_counters; counter++) +mmap_read (&mmap_array[i][counter]); }if (time (NULL ) >= last_refresh + delay_secs) {
[tip:perfcounters/core] perf_counter tools: tidy up in-kernel dependencies
Commit-ID: 4d1b100b57e8d472ca3b3a628e03c24d66cb782c Gitweb: http://git.kernel.org/tip/4d1b100b57e8d472ca3b3a628e03c24d66cb782c Author: Ingo Molnar <[email protected]> AuthorDate: Mon, 23 Mar 2009 21:49:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:51:35 +0100
perf_counter tools: tidy up in-kernel dependencies
Remove now unified perfstat.c and perf_counter.h, and link to the in-kernel perf_counter.h.
LKML-Reference: <[email protected]>
--- Documentation/perf_counter/Makefile | 2 +- Documentation/perf_counter/kerneltop.c | 2 +- Documentation/perf_counter/perfcounters.h | 142 ---------------- Documentation/perf_counter/perfstat.c | 251 ----------------------------- 4 files changed, 2 insertions(+), 395 deletions(-)
Documentation/perf_counter/Makefile +1 -1
@@ -2,7 +2,7 @@ BINS = kerneltop perfstat all: $(BINS) -kerneltop: kerneltop.c perfcounters.h +kerneltop: kerneltop.c ../../include/linux/perf_counter.h cc -O6 -Wall -lrt `pkg-config --cflags --libs glib-2.0` -o $@ $< perfstat: kerneltop
Documentation/perf_counter/kerneltop.c +1 -1
@@ -88,7 +88,7 @@#include <linux/unistd.h> -#include "include/linux/perf_counter.h" +#include "../../include/linux/perf_counter.h" /*
Documentation/perf_counter/perfcounters.h deleted +0 -142
142 lines not on this page · raw message
Documentation/perf_counter/perfstat.c deleted +0 -251
251 lines not on this page · raw message
Re: [tip:perfcounters/core] perf_counter tools: tidy up in-kernel dependencies
Quoted 31 lines · Ingo Molnar
Commit-ID: 4d1b100b57e8d472ca3b3a628e03c24d66cb782c Gitweb: http://git.kernel.org/tip/4d1b100b57e8d472ca3b3a628e03c24d66cb782c Author: Ingo Molnar <[email protected]> AuthorDate: Mon, 23 Mar 2009 21:49:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:51:35 +0100
perf_countertools: tidy up in-kernel dependenciesRemove now unified
perfstat.candperf_counter.h, and link to the in-kernelperf_counter.h.Cc Wu Fengguang <[email protected]>Cc Paul Mackerras <[email protected]>Acked-by Peter Zijlstra <[email protected]>LKML-Reference: <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>--- Documentation/perf_counter/Makefile | 2 +- Documentation/perf_counter/kerneltop.c | 2 +- Documentation/perf_counter/perfcounters.h | 142 ---------------- Documentation/perf_counter/perfstat.c | 251 ----------------------------- 4 files changed, 2 insertions(+), 395 deletions(-)Documentation/perf_counter/Makefile +0
-0
Documentation/perf_counter/Makefile BINS = kerneltop perfstat · line 8
Quoted 8 lines
Documentation/perf_counter/Makefile +1
-1@@ -2,7 +2,7 @@ BINS = kerneltop perfstat all: $(BINS) -kerneltop: kerneltop.c perfcounters.h +kerneltop: kerneltop.c ../../include/linux/perf_counter.h cc -O6 -Wall -lrt `pkg-config --cflags --libs glib-2.0` -o $@ $< perfstat: kerneltop
Can we support Documentation/perf_counter/Makefile to use Documentation/Makefile so that we can also get these perf_counter user apps along with new kernel.
--
JSR
[tip:perfcounters/core] kerneltop: use mmap() output
In reply to Peter ZijlstraCommit-ID: 0488f729768605502accdae7335911b4400a5927 Gitweb: http://git.kernel.org/tip/0488f729768605502accdae7335911b4400a5927 Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 23 Mar 2009 18:22:12 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 23 Mar 2009 21:45:11 +0100
kerneltop: use mmap() output
update kerneltop to use the mmap() output to gather overflow information
LKML-Reference: <[email protected]>
--- Documentation/perf_counter/kerneltop.c | 93 +++++++++++++++++++++++++++----- 1 files changed, 79 insertions(+), 14 deletions(-)
Documentation/perf_counter/kerneltop.c +79 -14
@@ -84,6 +84,7 @@#include <sys/prctl.h> #include <sys/wait.h> #include <sys/uio.h> +#include <sys/mman.h> #include <linux/unistd.h>
@@ -119,17 +120,25 @@ typedef long long __s64;#ifdef __x86_64__ -# define __NR_perf_counter_open 295 +#define __NR_perf_counter_open 295 +#define rmb () asm volatile("lfence" ::: "memory") +#define cpu_relax () asm volatile("rep; nop" ::: "memory");#endif #ifdef __i386__ -# define __NR_perf_counter_open 333 +#define __NR_perf_counter_open 333 +#define rmb () asm volatile("lfence" ::: "memory") +#define cpu_relax () asm volatile("rep; nop" ::: "memory");#endif #ifdef __powerpc__#define __NR_perf_counter_open 319 +#define rmb () asm volatile ("sync" ::: "memory") +#define cpu_relax () asm volatile ("" ::: "memory");#endif +#define unlikely (x) __builtin_expect(!!(x), 0) + asmlinkage int sys_perf_counter_open( struct perf_counter_hw_event *hw_event_uptr __user, pid_t pid,
@@ -181,6 +190,7 @@ static int profile_cpu = -1;static int nr_cpus =0 ;static int nmi =1 ;static int group =0 ; +static unsigned int page_size;static char *vmlinux;
@@ -1117,16 +1127,68 @@ static void process_options(int argc, char *argv[]) } } +struct mmap_data { +int counter; +void *base; +unsigned int mask; +unsigned int prev; +}; + +static unsigned int mmap_read_head (struct mmap_data *md) +{ +struct perf_counter_mmap_page *pc = md->base; +unsigned int seq, head; + +repeat: +rmb (); + seq = pc->lock; + +if (unlikely (seq &1 )) { +cpu_relax (); + goto repeat; + } + + head = pc->data_head; + +rmb (); +if (pc->lock != seq) + goto repeat; + +return head; +} + +static void mmap_read (struct mmap_data *md) +{ +unsigned int head =mmap_read_head (md); +unsigned int old = md->prev; +unsigned char *data = md->base + page_size; + +if (head - old > md->mask) { +printf ("ERROR: failed to keep up with mmap data\n" ); +exit (-1 ); + } + +for (; old != head;) { +__u64 *ptr = (__u64 *)&data[old & md->mask]; + old +=sizeof (__u64); + +process_event (*ptr, md->counter); + } + + md->prev = old; +} +int main (int argc,char *argv[]) {struct pollfd event_array[MAX_NR_CPUS ][MAX_COUNTERS ]; +struct mmap_data mmap_array[MAX_NR_CPUS ][MAX_COUNTERS ];struct perf_counter_hw_event hw_event;int i, counter, group_fd;unsigned int cpu; -uint64_t ip; -ssize_t res;int ret; + page_size =sysconf (_SC_PAGE_SIZE); +process_options (argc, argv); nr_cpus =sysconf (_SC_NPROCESSORS_ONLN);
@@ -1153,8 +1215,6 @@ int main(int argc, char *argv[]) hw_event.record_type =PERF_RECORD_IRQ ; hw_event.nmi = nmi; -printf ("FOO: %d %llx %llx\n" , counter, event_id[counter], event_count[counter]); - fd[i][counter] =sys_perf_counter_open (&hw_event, tid, cpu, group_fd,0 );fcntl (fd[i][counter],F_SETFL ,O_NONBLOCK );if (fd[i][counter] <0 ) {
@@ -1174,6 +1234,17 @@ int main(int argc, char *argv[]) event_array[i][counter].fd = fd[i][counter]; event_array[i][counter].events =POLLIN ; + + mmap_array[i][counter].counter = counter; + mmap_array[i][counter].prev =0 ; + mmap_array[i][counter].mask =2 *page_size -1 ; + mmap_array[i][counter].base =mmap (NULL ,3 *page_size, +PROT_READ ,MAP_SHARED , fd[i][counter],0 ); +if (mmap_array[i][counter].base ==MAP_FAILED ) { +printf ("kerneltop error: failed to mmap with %d (%s)\n" , + errno,strerror (errno)); +exit (-1 ); + } } }
@@ -1188,14 +1259,8 @@ int main(int argc, char *argv[])int hits = events;for (i =0 ; i < nr_cpus; i++) { -for (counter =0 ; counter < nr_counters; counter++) { - res =read (fd[i][counter], (char *) &ip,sizeof (ip)); -if (res >0 ) { -assert (res ==sizeof (ip)); - -process_event (ip, counter); - } - } +for (counter =0 ; counter < nr_counters; counter++) +mmap_read (&mmap_array[i][counter]); }if (time (NULL ) >= last_refresh + delay_secs) {