perf: Add cputime events/metrics
Subsystems (2)
Jiri Olsa
reviewerv2 cover perf: Add cputime events/metrics
hi,
so.. I failed to make work reliably the exclude_idle bit for cpu-clock event using the idle's process sum_exec_runtime as Peter outlined in his patch [1]. The time jumped up and down and I couldn't make it stable.
But I noticed we actually have IDLE stats (and many more) available for each CPU (enum cpu_usage_stat), we just can't reach them by perf yet.
This patchset adds 'cputime' perf software PMU, that provides CPUTIME_* stats via events that mirrors their names:
# perf list | grep cputime
cputime/guest/ [Kernel PMU event]
cputime/guest_nice/ [Kernel PMU event]
cputime/idle/ [Kernel PMU event]
cputime/iowait/ [Kernel PMU event]
cputime/irq/ [Kernel PMU event]
cputime/nice/ [Kernel PMU event]
cputime/softirq/ [Kernel PMU event]
cputime/steal/ [Kernel PMU event]
cputime/system/ [Kernel PMU event]
cputime/user/ [Kernel PMU event]v2 changes:
- all of the support patches are already in
- new way of 'fixing' of idle counts when tick is disabled (patch 2)
Examples:
# perf stat --top -I 1000
# time Idle System User Irq Softirq IO wait
1.001692690 100.0% 0.0% 0.0% 0.7% 0.2% 0.0%
2.002994039 98.9% 0.0% 0.0% 0.9% 0.2% 0.0%
3.004164038 98.5% 0.2% 0.2% 0.9% 0.2% 0.0%
4.005312773 98.9% 0.0% 0.0% 0.9% 0.2% 0.0% # perf stat --top-full -I 1000
# time Idle System User Irq Softirq IO wait Guest Guest nice Nice Steal
1.001750803 100.0% 0.0% 0.0% 0.7% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
2.003159490 99.0% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
3.004358366 99.0% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
4.005592436 98.9% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%# perf stat -e cpu-clock,cputime/system/,cputime/user/,cputime/idle/ -a sleep 10
Performance counter stats for 'system wide':
240070.828221 cpu-clock (msec) # 23.999 CPUs utilized
208,910,979,120 ns cputime/system/ # 87.0% System
20,589,603,359 ns cputime/user/ # 8.6% User
8,813,416,821 ns cputime/idle/ # 3.7% Idle 10.003261054 seconds time elapsed
# perf stat -e cpu-clock,cputime/system/,cputime/user/ yes > /dev/null ^Cyes: Interrupt
Performance counter stats for 'yes':
3483.824364 cpu-clock (msec) # 1.000 CPUs utilized
2,460,117,205 ns cputime/system/ # 70.6% System
1,018,360,669 ns cputime/user/ # 29.2% User 3.484554149 seconds time elapsed
1.018525000 seconds user
2.460515000 seconds sys# perf stat --top -I 1000 --interval-clear # perf stat --top -I 1000 --interval-clear --per-core # perf stat --top -I 1000 --interval-clear --per-socket # perf stat --top -I 1000 --interval-clear -A
It's also available in here:
git://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git
perf/fixes
thanks,
jirka
[1] https://marc.info/?l=linux-kernel&m=152397251027433&w=2
---
Jiri Olsa (3):
perf/cputime: Add cputime pmu
perf/cputime: Fix idle time on NO_HZ config perf stat: Add cputime metric support
include/linux/perf_event.h | 2 ++ include/linux/tick.h | 1 + kernel/events/Makefile | 2 +- kernel/events/core.c | 1 + kernel/events/cputime.c | 221 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/time/tick-sched.c | 11 ++++++ tools/perf/Documentation/perf-stat.txt | 65 +++++++++++++++++++++++++++++++++ tools/perf/builtin-stat.c | 47 ++++++++++++++++++++++++ tools/perf/util/stat-shadow.c | 72 +++++++++++++++++++++++++++++++++++++ tools/perf/util/stat.c | 10 ++++++ tools/perf/util/stat.h | 10 ++++++ 11 files changed, 441 insertions(+), 1 deletion(-) create mode 100644 kernel/events/cputime.c
Jiri Olsa
reviewerPATCH v2 1/3 perf/cputime: Add cputime pmu
The CPUTIME_* counters account the time for CPU runtimes. Adding 'cputime' PMU, that provides perf interface to those counters.
The 'cputime' interface is standard software PMU, that provides following events, meassuring their CPUTIME counterparts:
PERF_CPUTIME_USER - CPUTIME_USER PERF_CPUTIME_NICE - CPUTIME_NICE PERF_CPUTIME_SYSTEM - CPUTIME_SYSTEM PERF_CPUTIME_SOFTIRQ - CPUTIME_SOFTIRQ PERF_CPUTIME_IRQ - CPUTIME_IRQ PERF_CPUTIME_IDLE - CPUTIME_IDLE PERF_CPUTIME_IOWAIT - CPUTIME_IOWAIT PERF_CPUTIME_STEAL - CPUTIME_STEAL PERF_CPUTIME_GUEST - CPUTIME_GUEST
PERF_CPUTIME_GUEST_NICE - CPUTIME_GUEST_NICE
The 'cputime' PMU adds 'events' and 'format' directory, with above events specifics.
It can be used via perf tool like:
# perf stat -e cputime/system/,cputime/user/ yes > /dev/null ^Cyes: Interrupt
Performance counter stats for 'yes':
2,177,550,368 ns cputime/system/
567,029,895 ns cputime/user/
2.749451438 seconds time elapsed
0.567127000 seconds user
2.177924000 seconds sysSigned-off-by Jiri Olsa <[email protected]>
--- include/linux/perf_event.h | 2 + kernel/events/Makefile | 2 +- kernel/events/core.c | 1 + kernel/events/cputime.c | 198 +++++++++++++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 kernel/events/cputime.c
include/linux/perf_event.h +2 -0
@@ -1162,6 +1162,8 @@ static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 } } +extern int __init perf_cputime_register (void ); +extern int sysctl_perf_event_paranoid;extern int sysctl_perf_event_mlock;extern int sysctl_perf_event_sample_rate;
kernel/events/Makefile +1 -1
@@ -3,7 +3,7 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_core.o = $(CC_FLAGS_FTRACE) endif -obj-y := core.o ring_buffer.o callchain.o +obj-y := core.o ring_buffer.o callchain.o cputime.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_UPROBES) += uprobes.o
kernel/events/core.c +1 -0
@@ -11723,6 +11723,7 @@ void __init perf_event_init(void)perf_pmu_register (&perf_cpu_clock,NULL ,-1 );perf_pmu_register (&perf_task_clock,NULL ,-1 );perf_tp_register (); +perf_cputime_register ();perf_event_init_cpu (smp_processor_id ());register_reboot_notifier (&perf_reboot_notifier);
kernel/events/cputime.c new file +198 -0
198 lines not on this page · raw message
Signature · 1 line
2.17.2
Jiri Olsa
reviewerPATCH v2 2/3 perf/cputime: Fix idle time on NO_HZ config
In reply to Jiri OlsaIn case there's NO_HZ enabled we won't get proper numbers for idle counter if the tick is disabled on the cpu.
Making up for it by counting the idle counter value if the tick is stopped, which will keep the counter number updated at the time it is read.
Signed-off-by Jiri Olsa <[email protected]>
--- include/linux/tick.h | 1 + kernel/events/cputime.c | 25 ++++++++++++++++++++++++- kernel/time/tick-sched.c | 11 +++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-)
include/linux/tick.h +1 -0
@@ -125,6 +125,7 @@ extern bool tick_nohz_idle_got_tick(void);extern ktime_t tick_nohz_get_sleep_length (ktime_t *delta_next);extern unsigned long tick_nohz_get_idle_calls (void );extern unsigned long tick_nohz_get_idle_calls_cpu (int cpu); +extern unsigned long tick_nohz_get_idle_jiffies_cpu (int cpu);extern u64 get_cpu_idle_time_us (int cpu,u64 *last_update_time);extern u64 get_cpu_iowait_time_us (int cpu,u64 *last_update_time);
kernel/events/cputime.c +24 -1
@@ -1,6 +1,7 @@#include <linux/kernel_stat.h> #include <linux/sched.h> #include <linux/perf_event.h> +#include <linux/tick.h> enum perf_cputime_id {PERF_CPUTIME_USER ,
@@ -102,11 +103,33 @@ static const struct attribute_group *cputime_attr_groups[] = { NULL, }; +#ifdef CONFIG_NO_HZ_COMMON +static u64 idle_fix (int cpu) +{ +u64 ticks; + +if (!tick_nohz_tick_stopped_cpu (cpu)) +return 0 ; + + ticks = jiffies -tick_nohz_get_idle_jiffies_cpu (cpu); +return ticks *TICK_NSEC ; +} +#else +static u64 idle_fix (int cpu) +{ +return 0 ; +} +#endif +static u64 cputime_read_counter (struct perf_event *event) {int cpu = event->oncpu; +u64 val =kcpustat_cpu (cpu).cpustat[event->hw.config]; + +if (event->hw.config ==PERF_CPUTIME_IDLE ) + val +=idle_fix (cpu); -return kcpustat_cpu (cpu).cpustat[event->hw.config]; +return val; }static void perf_cputime_update (struct perf_event *event)
kernel/time/tick-sched.c +11 -0
@@ -1077,6 +1077,17 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu)return ts->idle_calls; } +/** +* tick_nohz_get_idle_jiffies_cpu - return the current idle jiffies counter value +* for a particular CPU. +*/ +unsigned long tick_nohz_get_idle_jiffies_cpu (int cpu) +{ +struct tick_sched *ts =tick_get_tick_sched (cpu); + +return ts->idle_jiffies; +} +/** * tick_nohz_get_idle_calls - return the current idle calls counter value *
Signature · 1 line
2.17.2
Jiri Olsa
reviewerPATCH v2 3/3 perf stat: Add cputime metric support
In reply to Jiri OlsaAdding --top/--top-full options to provide metrics based on the cputime PMU events. Simply all the metrics are simple ratios of events to STAT_NSECS time to get their % value.
The --top option provides basic subset of cputime metrics:
# perf stat --top -I 1000
# time Idle System User Irq Softirq IO wait
1.001692690 100.0% 0.0% 0.0% 0.7% 0.2% 0.0%
2.002994039 98.9% 0.0% 0.0% 0.9% 0.2% 0.0%
3.004164038 98.5% 0.2% 0.2% 0.9% 0.2% 0.0%
4.005312773 98.9% 0.0% 0.0% 0.9% 0.2% 0.0%The --top-full option provides all cputime metrics:
# perf stat --top-full -I 1000
# time Idle System User Irq Softirq IO wait Guest Guest nice Nice Steal
1.001750803 100.0% 0.0% 0.0% 0.7% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
2.003159490 99.0% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
3.004358366 99.0% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%
4.005592436 98.9% 0.0% 0.0% 0.9% 0.2% 0.0% 0.0% 0.0% 0.0% 0.0%Signed-off-by Jiri Olsa <[email protected]>
--- tools/perf/Documentation/perf-stat.txt | 65 +++++++++++++++++++++++ tools/perf/builtin-stat.c | 47 +++++++++++++++++ tools/perf/util/stat-shadow.c | 72 ++++++++++++++++++++++++++ tools/perf/util/stat.c | 10 ++++ tools/perf/util/stat.h | 10 ++++ 5 files changed, 204 insertions(+)
tools/perf/Documentation/perf-stat.txt +65 -0
@@ -310,6 +310,71 @@ The output is SMI cycles%, equals to (aperf - unhalted core cycles) / aperf Users who wants to get the actual value can apply --no-metric-only. +--top:: +--top-full: +Measure cputime PMU events and display percentage of CPU utilization rates. + +The --top option displays rates for following events: + idle system user irq softirq iowait + +The --top-full option displays additional rates: + guest guest_nice nice steal + +Examples: + # perf stat --top + ^C + Performance counter stats for 'system wide': + + Idle System User Irq Softirq IO wait + 1.3% 89.5% 7.4% 1.8% 0.1% 0.0% + + 7.282332605 seconds time elapsed + + # perf stat --top-full + ^C + Performance counter stats for 'system wide': + + Idle System User Irq Softirq IO wait Guest Guest nice Nice Steal + 5.4% 85.4% 8.6% 0.5% 0.1% 0.0% 0.0% 0.0% 0.0% 0.0% + + 7.618359683 seconds time elapsed + + # perf stat --top -I 1000 + # time Idle System User Irq Softirq IO wait + 1.000525839 5.4% 85.3% 8.8% 0.4% 0.1% 0.0% + 2.001032632 5.1% 85.7% 8.7% 0.4% 0.1% 0.0% + 3.001388414 5.2% 85.7% 8.6% 0.4% 0.1% 0.0% + 4.001758697 5.7% 85.2% 8.6% 0.5% 0.1% 0.0% + + # perf stat --top -I 1000 -A + # time CPU Idle System User Irq Softirq IO wait + 1.000485174 CPU0 6.9% 84.0% 8.6% 0.5% 0.1% 0.0% + 1.000485174 CPU1 5.5% 84.8% 9.1% 0.5% 0.1% 0.0% + 1.000485174 CPU2 5.5% 86.6% 7.4% 0.5% 0.1% 0.0% + ... + + # perf stat --top -I 1000 --per-core + # time core cpus Idle System User Irq Softirq IO wait + 1.000450719 S0-C0 2 4.6% 87.0% 7.9% 0.4% 0.1% 0.0% + 1.000450719 S0-C1 2 4.8% 86.3% 8.3% 0.4% 0.1% 0.0% + 1.000450719 S0-C2 2 5.3% 86.3% 7.8% 0.4% 0.1% 0.0% + 1.000450719 S0-C3 2 5.2% 85.5% 8.7% 0.4% 0.1% 0.0% + 1.000450719 S0-C4 2 4.5% 86.7% 8.3% 0.4% 0.1% 0.0% + + # perf stat --top ./perf bench sched messaging -l 10000 + ... + Total time: 7.089 [sec] + + Performance counter stats for './perf bench sched messaging -l 10000': + + Idle System User Irq Softirq IO wait + 0.0% 90.1% 8.9% 0.5% 0.1% 0.0% + + 7.186366800 seconds time elapsed + + 14.527066000 seconds user + 146.254278000 seconds sys + EXAMPLES --------
tools/perf/builtin-stat.c +47 -0
71 lines not on this page · raw message
tools/perf/util/stat-shadow.c +72 -0
84 lines not on this page · raw message
tools/perf/util/stat.c +10 -0
@@ -89,6 +89,16 @@ static const char *id_str[PERF_STAT_EVSEL_ID__MAX] = {ID (TOPDOWN_RECOVERY_BUBBLES , topdown-recovery-bubbles), ID(SMI_NUM, msr/smi/), ID(APERF, msr/aperf/), + ID(CPUTIME_GUEST, cputime/guest/), + ID(CPUTIME_GUEST_NICE, cputime/guest_nice/), + ID(CPUTIME_IDLE, cputime/idle/), + ID(CPUTIME_IOWAIT, cputime/iowait/), + ID(CPUTIME_IRQ, cputime/irq/), + ID(CPUTIME_NICE, cputime/nice/), + ID(CPUTIME_SOFTIRQ, cputime/softirq/), + ID(CPUTIME_STEAL, cputime/steal/), + ID(CPUTIME_SYSTEM, cputime/system/), + ID(CPUTIME_USER, cputime/user/), };#undef ID
tools/perf/util/stat.h +10 -0
@@ -31,6 +31,16 @@ enum perf_stat_evsel_id {PERF_STAT_EVSEL_ID__TOPDOWN_RECOVERY_BUBBLES ,PERF_STAT_EVSEL_ID__SMI_NUM ,PERF_STAT_EVSEL_ID__APERF , +PERF_STAT_EVSEL_ID__CPUTIME_GUEST , +PERF_STAT_EVSEL_ID__CPUTIME_GUEST_NICE , +PERF_STAT_EVSEL_ID__CPUTIME_IDLE , +PERF_STAT_EVSEL_ID__CPUTIME_IOWAIT , +PERF_STAT_EVSEL_ID__CPUTIME_IRQ , +PERF_STAT_EVSEL_ID__CPUTIME_NICE , +PERF_STAT_EVSEL_ID__CPUTIME_SOFTIRQ , +PERF_STAT_EVSEL_ID__CPUTIME_STEAL , +PERF_STAT_EVSEL_ID__CPUTIME_SYSTEM , +PERF_STAT_EVSEL_ID__CPUTIME_USER , PERF_STAT_EVSEL_ID__MAX, };
Signature · 1 line
2.17.2