Skip to contentSkip to conversation

perf diff: Report noisy for cycles diff

PATCH Superseded Jin Yao 2 participants 5 messages Aug 13, 2019 – Aug 16, 2019 perf events

v4 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Jin Yao

author
#0 lore raw

PATCH v4 1/1 perf diff: Report noisy for cycles diff

This patch prints the stddev and hist for the cycles diff of program block. It can help us to understand if the cycles is noisy or not.

This patch is inspired by Andi Kleen's patch
https://lwn.net/Articles/600471/

We create new option '--cycles-hist'.

Example:

perf record -b ./div
perf record -b ./div
perf diff -c cycles

  # Baseline                                       [Program Block Range] Cycles Diff  Shared Object      Symbol
  # ........  ......................................................................  .................  ............................
  #
      46.72%                                             [div.c:40 -> div.c:40]    0  div                [.] main
      46.72%                                             [div.c:42 -> div.c:44]    0  div                [.] main
      46.72%                                             [div.c:42 -> div.c:39]    0  div                [.] main
      20.54%                                 [random_r.c:357 -> random_r.c:394]    1  libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:357 -> random_r.c:380]    0  libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:388 -> random_r.c:388]    0  libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:388 -> random_r.c:391]    0  libc-2.27.so       [.] __random_r
      17.04%                                     [random.c:288 -> random.c:291]    0  libc-2.27.so       [.] __random
      17.04%                                     [random.c:291 -> random.c:291]    0  libc-2.27.so       [.] __random
      17.04%                                     [random.c:293 -> random.c:293]    0  libc-2.27.so       [.] __random
      17.04%                                     [random.c:295 -> random.c:295]    0  libc-2.27.so       [.] __random
      17.04%                                     [random.c:295 -> random.c:295]    0  libc-2.27.so       [.] __random
      17.04%                                     [random.c:298 -> random.c:298]    0  libc-2.27.so       [.] __random
       8.40%                                             [div.c:22 -> div.c:25]    0  div                [.] compute_flag
       8.40%                                             [div.c:27 -> div.c:28]    0  div                [.] compute_flag
       5.14%                                           [rand.c:26 -> rand.c:27]    0  libc-2.27.so       [.] rand
       5.14%                                           [rand.c:28 -> rand.c:28]    0  libc-2.27.so       [.] rand
       2.15%                                         [rand@plt+0 -> rand@plt+0]    0  div                [.] rand@plt
       0.00%                                                                          [kernel.kallsyms]  [k] __x86_indirect_thunk_rax
       0.00%                                       [do_mmap+714 -> do_mmap+732]  -10  [kernel.kallsyms]  [k] do_mmap
       0.00%                                       [do_mmap+737 -> do_mmap+765]    1  [kernel.kallsyms]  [k] do_mmap
       0.00%                                       [do_mmap+262 -> do_mmap+299]    0  [kernel.kallsyms]  [k] do_mmap
       0.00%         [__x86_indirect_thunk_r15+0 -> __x86_indirect_thunk_r15+0]    7  [kernel.kallsyms]  [k] __x86_indirect_thunk_r15
       0.00%                   [native_sched_clock+0 -> native_sched_clock+119]   -1  [kernel.kallsyms]  [k] native_sched_clock
       0.00%                        [native_write_msr+0 -> native_write_msr+16]  -13  [kernel.kallsyms]  [k] native_write_msr

When we enable the option '--cycles-hist', the output is

perf diff -c cycles --cycles-hist

  # Baseline                                       [Program Block Range] Cycles Diff        stddev/Hist  Shared Object      Symbol
  # ........  ......................................................................  .................  .................  ............................
  #
      46.72%                                             [div.c:40 -> div.c:40]    0  ± 37.8% ▁█▁▁██▁█   div                [.] main
      46.72%                                             [div.c:42 -> div.c:44]    0  ± 49.4% ▁▁▂█▂▂▂▂   div                [.] main
      46.72%                                             [div.c:42 -> div.c:39]    0  ± 24.1% ▃█▂▄▁▃▂▁   div                [.] main
      20.54%                                 [random_r.c:357 -> random_r.c:394]    1  ± 33.5% ▅▂▁█▃▁▂▁   libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:357 -> random_r.c:380]    0  ± 39.4% ▁▁█▁██▅▁   libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:388 -> random_r.c:388]    0                     libc-2.27.so       [.] __random_r
      20.54%                                 [random_r.c:388 -> random_r.c:391]    0  ± 41.2% ▁▃▁▂█▄▃▁   libc-2.27.so       [.] __random_r
      17.04%                                     [random.c:288 -> random.c:291]    0  ± 48.8% ▁▁▁▁███▁   libc-2.27.so       [.] __random
      17.04%                                     [random.c:291 -> random.c:291]    0  ±100.0% ▁█▁▁▁▁▁▁   libc-2.27.so       [.] __random
      17.04%                                     [random.c:293 -> random.c:293]    0  ±100.0% ▁█▁▁▁▁▁▁   libc-2.27.so       [.] __random
      17.04%                                     [random.c:295 -> random.c:295]    0  ±100.0% ▁█▁▁▁▁▁▁   libc-2.27.so       [.] __random
      17.04%                                     [random.c:295 -> random.c:295]    0                     libc-2.27.so       [.] __random
      17.04%                                     [random.c:298 -> random.c:298]    0  ± 75.6% ▃█▁▁▁▁▁▁   libc-2.27.so       [.] __random
       8.40%                                             [div.c:22 -> div.c:25]    0  ± 42.1% ▁▃▁▁███▁   div                [.] compute_flag
       8.40%                                             [div.c:27 -> div.c:28]    0  ± 41.8% ██▁▁▄▁▁▄   div                [.] compute_flag
       5.14%                                           [rand.c:26 -> rand.c:27]    0  ± 37.8% ▁▁▁████▁   libc-2.27.so       [.] rand
       5.14%                                           [rand.c:28 -> rand.c:28]    0                     libc-2.27.so       [.] rand
       2.15%                                         [rand@plt+0 -> rand@plt+0]    0                     div                [.] rand@plt
       0.00%                                                                                             [kernel.kallsyms]  [k] __x86_indirect_thunk_rax
       0.00%                                       [do_mmap+714 -> do_mmap+732]  -10                     [kernel.kallsyms]  [k] do_mmap
       0.00%                                       [do_mmap+737 -> do_mmap+765]    1                     [kernel.kallsyms]  [k] do_mmap
       0.00%                                       [do_mmap+262 -> do_mmap+299]    0                     [kernel.kallsyms]  [k] do_mmap
       0.00%         [__x86_indirect_thunk_r15+0 -> __x86_indirect_thunk_r15+0]    7                     [kernel.kallsyms]  [k] __x86_indirect_thunk_r15
       0.00%                   [native_sched_clock+0 -> native_sched_clock+119]   -1  ± 38.5% ▄█▁        [kernel.kallsyms]  [k] native_sched_clock
       0.00%                        [native_write_msr+0 -> native_write_msr+16]  -13  ± 47.1% ▁█▇▃▁▁     [kernel.kallsyms]  [k] native_write_msr

v4:
---
1. Rename the new option from '--noisy' to '--cycles-hist'
2. Remove the option '-n'.
3. Only update the spark value and stats when '--cycles-hist' is enabled.
4. Remove the code of printing '..'.

v3:
---
1. Move the histogram to a separate column
2. Move the svals[] out of struct stats

v2:
---
Jiri got a compile error,

CC builtin-diff.o
builtin-diff.c: In function ‘compute_cycles_diff’:
builtin-diff.c:712:10: error: taking the absolute value of unsigned type ‘u64’ {aka ‘long unsigned int’} has no effect [-Werror=absolute-value]

  712 |          labs(pair->block_info->cycles_spark[i] -
      |          ^~~~

Because the result of u64 - u64 is still u64. Now we change the type of cycles_spark[] to s64.

---
 tools/perf/Documentation/perf-diff.txt |   5 +
 tools/perf/builtin-diff.c              | 150 ++++++++++++++++++++++++-
 tools/perf/util/Build                  |   1 +
 tools/perf/util/annotate.c             |   4 +
 tools/perf/util/annotate.h             |   2 +
 tools/perf/util/sort.h                 |   4 +
 tools/perf/util/spark.c                |  34 ++++++
 tools/perf/util/spark.h                |   8 ++
 tools/perf/util/symbol.h               |   2 +
 9 files changed, 208 insertions(+), 2 deletions(-)
 create mode 100644 tools/perf/util/spark.c
 create mode 100644 tools/perf/util/spark.h
tools/perf/Documentation/perf-diff.txt +5 -0
index d5cc15e651cf..f50ca0fef0a4 100644
@@ -95,6 +95,11 @@ OPTIONS
         diff.compute config option.  See COMPARISON METHODS section for
         more info.
 
+--cycles-hist::
+	Report a histogram and the standard deviation for cycles data.
+	It can help us to judge if the reported cycles data is noisy or
+	not. This option should be used with '-c cycles'.
+
 -p::
 --period::
         Show period values for both compared hist entries.
tools/perf/builtin-diff.c +148 -2

222 lines not on this page · raw message

tools/perf/util/Build +1 -0
index 7abf05131889..8dc5b1dd9c5a 100644
@@ -89,6 +89,7 @@ perf-y += cloexec.o
 perf-y += call-path.o
 perf-y += rwsem.o
 perf-y += thread-stack.o
+perf-y += spark.o
 perf-$(CONFIG_AUXTRACE) += auxtrace.o
 perf-$(CONFIG_AUXTRACE) += intel-pt-decoder/
 perf-$(CONFIG_AUXTRACE) += intel-pt.o
tools/perf/util/annotate.c +4 -0
index e0518dc4c4d2..a43354dbc836 100644
@@ -847,6 +847,10 @@ static int __symbol__account_cycles(struct cyc_hist *ch,
 			   ch[offset].start < start)
 			return 0;
 	}
+
+	if (ch[offset].num < NUM_SPARKS)
+		ch[offset].cycles_spark[ch[offset].num] = cycles;
+
 	ch[offset].have_start = have_start;
 	ch[offset].start = start;
 	ch[offset].cycles += cycles;
tools/perf/util/annotate.h +2 -0
index d94be9140e31..7776ff4480af 100644
@@ -11,6 +11,7 @@
 #include <pthread.h>
 #include <asm/bug.h>
 #include "symbol_conf.h"
+#include "spark.h"
 
 struct hist_browser_timer;
 struct hist_entry;
@@ -235,6 +236,7 @@ struct cyc_hist {
 	u64	cycles_aggr;
 	u64	cycles_max;
 	u64	cycles_min;
+	s64	cycles_spark[NUM_SPARKS];
 	u32	num;
 	u32	num_aggr;
 	u8	have_start;
tools/perf/util/sort.h +4 -0
index 5e34676a98e8..14061e42a862 100644
@@ -23,6 +23,8 @@
 #include "parse-events.h"
 #include "hist.h"
 #include "srcline.h"
+#include "stat.h"
+#include "spark.h"
 
 struct thread;
 
@@ -83,6 +85,8 @@ struct hist_entry_diff {
 		/* PERF_HPP_DIFF__CYCLES */
 		s64	cycles;
 	};
+	struct stats	stats;
+	unsigned long	svals[NUM_SPARKS];
 };
 
 struct hist_entry_ops {
tools/perf/util/spark.c new file +34 -0
new file mode 100644
index 000000000000..70272a8b81a6
@@ -0,0 +1,34 @@
+#include <stdio.h>
+#include <limits.h>
+#include <string.h>
+#include <stdlib.h>
+#include "spark.h"
+#include "stat.h"
+
+#define SPARK_SHIFT 8
+
+/* Print spark lines on outf for numval values in val. */
+int print_spark(char *bf, int size, unsigned long *val, int numval)
+{
+	static const char *ticks[NUM_SPARKS] = {
+		"▁",  "▂", "▃", "▄", "▅", "▆", "▇", "█"
+	};
+	int i, printed = 0;
+	unsigned long min = ULONG_MAX, max = 0, f;
+
+	for (i = 0; i < numval; i++) {
+		if (val[i] < min)
+			min = val[i];
+		if (val[i] > max)
+			max = val[i];
+	}
+	f = ((max - min) << SPARK_SHIFT) / (NUM_SPARKS - 1);
+	if (f < 1)
+		f = 1;
+	for (i = 0; i < numval; i++) {
+		printed += scnprintf(bf + printed, size - printed, "%s",
+				     ticks[((val[i] - min) << SPARK_SHIFT) / f]);
+	}
+
+	return printed;
+}
tools/perf/util/spark.h new file +8 -0
new file mode 100644
index 000000000000..25402d7d7a64
@@ -0,0 +1,8 @@
+#ifndef SPARK_H
+#define SPARK_H 1
+
+#define NUM_SPARKS 8
+
+int print_spark(char *bf, int size, unsigned long *val, int numval);
+
+#endif
tools/perf/util/symbol.h +2 -0
index 12755b42ea93..ca4d55b68be4 100644
@@ -12,6 +12,7 @@
 #include "branch.h"
 #include "path.h"
 #include "symbol_conf.h"
+#include "spark.h"
 
 #ifdef HAVE_LIBELF_SUPPORT
 #include <libelf.h>
@@ -137,6 +138,7 @@ struct block_info {
 	u64			end;
 	u64			cycles;
 	u64			cycles_aggr;
+	s64			cycles_spark[NUM_SPARKS];
 	int			num;
 	int			num_aggr;
 	refcount_t		refcnt;
Signature · 1 line
2.17.1
#1 lore raw

On Tue, Aug 13, 2019 at 03:30:37PM +0800, Jin Yao wrote:

SNIP

Quoted 19 lines

static void
hpp__entry_unpair(struct hist_entry *he, int idx, char *buf, size_t size) {

tools/perf/builtin-diff.c +4 -1
@@ -1662,6 +1794,10 @@ static void data__hpp_register(struct data__file *d, int idx)
 		fmt->color = hpp__color_cycles;
 		fmt->sort  = hist_entry__cmp_nop;
 		break;
+	case PERF_HPP_DIFF__CYCLES_HIST:
+		fmt->color = hpp__color_cycles_hist;
+		fmt->sort  = hist_entry__cmp_nop;
+		break;
 	default:
 		fmt->sort  = hist_entry__cmp_nop;
 		break;
@@ -1688,8 +1824,15 @@ static int ui_init(void)
 		 *   PERF_HPP_DIFF__RATIO
 		 *   PERF_HPP_DIFF__WEIGHTED_DIFF
 		 */
-		data__hpp_register(d, i ? compute_2_hpp[compute] :

tools/perf/builtin-diff.c static int ui_init(void) · line 1835

Quoted 10 lines
tools/perf/builtin-diff.c +9 -1
-					  PERF_HPP_DIFF__BASELINE);
+		if (cycles_hist && (compute == COMPUTE_CYCLES)) {
+			data__hpp_register(d, i ? PERF_HPP_DIFF__CYCLES :
+						  PERF_HPP_DIFF__BASELINE);
+			data__hpp_register(d, i ? PERF_HPP_DIFF__CYCLES_HIST :
+						  PERF_HPP_DIFF__BASELINE);
+		} else {
+			data__hpp_register(d, i ? compute_2_hpp[compute] :
+						  PERF_HPP_DIFF__BASELINE);
+		}

I tink that something like this might be less confusing instead of above:

                if (cycles_hist && i && (compute == COMPUTE_CYCLES)) 
                        data__hpp_register(d, PERF_HPP_DIFF__CYCLES);

other than that the patch looks ok to me

jirka

#3 lore raw
In reply to Jin Yao
Quoted 6 lines · Jin Yao

This patch prints the stddev and hist for the cycles diff of program block. It can help us to understand if the cycles is noisy or not.

This patch is inspired by Andi Kleen's patch
https://lwn.net/Articles/600471/

now that we have sparks support, we could respin it ;-)

jirka

author
#2 lore raw
In reply to Jiri Olsa
Quoted 36 lines · Jiri Olsa

On Tue, Aug 13, 2019 at 03:30:37PM +0800, Jin Yao wrote:

SNIP

Quoted 19 lines
  static void
  hpp__entry_unpair(struct hist_entry *he, int idx, char *buf, size_t size)
  {
tools/perf/builtin-diff.c +4 -1
@@ -1662,6 +1794,10 @@ static void data__hpp_register(struct data__file *d, int idx)
  		fmt->color = hpp__color_cycles;
  		fmt->sort  = hist_entry__cmp_nop;
  		break;
+	case PERF_HPP_DIFF__CYCLES_HIST:
+		fmt->color = hpp__color_cycles_hist;
+		fmt->sort  = hist_entry__cmp_nop;
+		break;
  	default:
  		fmt->sort  = hist_entry__cmp_nop;
  		break;
@@ -1688,8 +1824,15 @@ static int ui_init(void)
  		 *   PERF_HPP_DIFF__RATIO
  		 *   PERF_HPP_DIFF__WEIGHTED_DIFF
  		 */
-		data__hpp_register(d, i ? compute_2_hpp[compute] :

tools/perf/builtin-diff.c static int ui_init(void) · line 1835

Quoted 10 lines
tools/perf/builtin-diff.c +9 -1
-					  PERF_HPP_DIFF__BASELINE);
+		if (cycles_hist && (compute == COMPUTE_CYCLES)) {
+			data__hpp_register(d, i ? PERF_HPP_DIFF__CYCLES :
+						  PERF_HPP_DIFF__BASELINE);
+			data__hpp_register(d, i ? PERF_HPP_DIFF__CYCLES_HIST :
+						  PERF_HPP_DIFF__BASELINE);
+		} else {
+			data__hpp_register(d, i ? compute_2_hpp[compute] :
+						  PERF_HPP_DIFF__BASELINE);
+		}

I tink that something like this might be less confusing instead of above:

Quoted 7 lines
                 if (cycles_hist && i && (compute == COMPUTE_CYCLES))
                         data__hpp_register(d, PERF_HPP_DIFF__CYCLES);

other than that the patch looks ok to me

jirka

Yes, your code is more beautiful. Thanks so much! :)

Thanks
Jin Yao

author
#4 lore raw
In reply to Jiri Olsa
Quoted 5 lines · Jiri Olsa
Quoted 4 lines · Jin Yao

This patch prints the stddev and hist for the cycles diff of program block. It can help us to understand if the cycles is noisy or not.

Quoted 7 lines
Quoted 2 lines

This patch is inspired by Andi Kleen's patch
https://lwn.net/Articles/600471/

now that we have sparks support, we could respin it ;-)

jirka

The sparks can be used in more places for reporting the data noisy. :)

Thanks
Jin Yao