Skip to contentSkip to conversation

perf,tools: move callchain option parse code to util.c

RFC PATCH In review [email protected] 4 participants 20 messages Aug 7, 2015 – Aug 12, 2015 perf events

lore
Share & tools
lore
#0 lore raw

RFC PATCH v9 1/3 perf,tools: move callchain option parse code to util.c

From: Kan Liang <[email protected]>

Move callchain option parse related code to util.c

---

Changes since V8:
- Fix the link problems found by perf python test case. Introduce the patch to move callchain option parse related code to util.c

 tools/perf/util/callchain.c | 89 +-------------------------------------------
 tools/perf/util/callchain.h |  1 +
 tools/perf/util/util.c      | 90 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/util.h      |  2 +
 4 files changed, 94 insertions(+), 88 deletions(-)
tools/perf/util/callchain.c +1 -88
index 931cca8..773fe13 100644
@@ -25,96 +25,9 @@
 
 __thread struct callchain_cursor callchain_cursor;
 
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-static int get_stack_size(const char *str, unsigned long *_size)
-{
-	char *endptr;
-	unsigned long size;
-	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
-
-	size = strtoul(str, &endptr, 0);
-
-	do {
-		if (*endptr)
-			break;
-
-		size = round_up(size, sizeof(u64));
-		if (!size || size > max_size)
-			break;
-
-		*_size = size;
-		return 0;
-
-	} while (0);
-
-	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
-	       max_size, str);
-	return -1;
-}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param)
 {
-	char *tok, *name, *saveptr = NULL;
-	char *buf;
-	int ret = -1;
-
-	/* We need buffer that we know we can write to. */
-	buf = malloc(strlen(arg) + 1);
-	if (!buf)
-		return -ENOMEM;
-
-	strcpy(buf, arg);
-
-	tok = strtok_r((char *)buf, ",", &saveptr);
-	name = tok ? : (char *)buf;
-
-	do {
-		/* Framepointer style */
-		if (!strncmp(name, "fp", sizeof("fp"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_FP;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-				       "needed for --call-graph fp\n");
-			break;
-
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-		/* Dwarf style */
-		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
-			const unsigned long default_stack_dump_size = 8192;
-
-			ret = 0;
-			param->record_mode = CALLCHAIN_DWARF;
-			param->dump_size = default_stack_dump_size;
-
-			tok = strtok_r(NULL, ",", &saveptr);
-			if (tok) {
-				unsigned long size = 0;
-
-				ret = get_stack_size(tok, &size);
-				param->dump_size = size;
-			}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_LBR;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-					"needed for --call-graph lbr\n");
-			break;
-		} else {
-			pr_err("callchain: Unknown --call-graph option "
-			       "value: %s\n", arg);
-			break;
-		}
-
-	} while (0);
-
-	free(buf);
-	return ret;
+	return parse_callchain_record(arg, param);
 }
 
 static int parse_callchain_mode(const char *value)
tools/perf/util/callchain.h +1 -0
index 68a32c2..acee2b3 100644
@@ -177,6 +177,7 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *
 			bool hide_unresolved);
 
 extern const char record_callchain_help[];
+extern int parse_callchain_record(const char *arg, struct callchain_param *param);
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param);
 int parse_callchain_report_opt(const char *arg);
 int perf_callchain_config(const char *var, const char *value);
tools/perf/util/util.c +90 -0

96 lines not on this page · raw message

tools/perf/util/util.h +2 -0
index 20d625a..8148703 100644
@@ -351,4 +351,6 @@ static inline char *asprintf_expr_not_in_ints(const char *var, size_t nints, int
 	return asprintf_expr_inout_ints(var, false, nints, ints);
 }
 
+int get_stack_size(const char *str, unsigned long *_size);
+
 #endif /* GIT_COMPAT_UTIL_H */
Signature · 1 line
1.8.3.1
#1 lore raw

RFC PATCH v9 2/3 perf,tools: per-event callgraph support

From: Kan Liang <[email protected]>

When multiple events are sampled it may not be needed to collect callgraphs for all of them. The sample sites are usually nearby, and it's enough to collect the callgraphs on a reference event (such as precise cycles or precise instructions).
This patchkit adds the ability to turn off callgraphs and time stamp per event. This in term can reduce sampling overhead and the size of the perf.data. Furthermore, it makes collecting back traces and timestamps possible when PEBS threshold > 1, which significantly reducing the sampling overhead especially for frequently occurring events
(https://lkml.org/lkml/2015/5/10/196). For example, A slower event with a larger period collects back traces/timestamps. Other more events run fast with multi-pebs. The time stamps from the slower events can be used to order the faster events. Their backtraces can give the user enough hint to find the right spot.

Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

The test command for PARTIAL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,call-graph=fp,time,period=100000/,

     cpu/instructions,call-graph=no,time=0,period=20000/p}'"

The elapsed time for FULL is 24.3 Sec, while for PARTIAL is 16.9 Sec. The perf.data size for FULL is 22.1 Gb, while for PARTIAL is 12.4 Gb.

2. Comparing the perf.data size and callgraph results.

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/pp,cpu/instructions,period=20000/p}' --call-graph fp -- ./tchain_edit"

The test command for PARTIAL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,call-graph=fp,time,period=100000/pp,

     cpu/instructions,call-graph=no,time=0,period=20000/p}'

-- ./tchain_edit"

The perf.data size for FULL is 43.2 MB, while for PARTIAL is 21.1 MB. The callgraph is roughly the same.

The callgraph from FULL
# Samples: 87K of event 'cpu/cpu-cycles,call-graph=fp,time,period=100000/pp' # Event count (approx.): 8760000000
#

 # Children      Self  Command      Shared Object       Symbol
 # ........  ........  ...........  ..................
..........................................
 #
    99.98%     0.00%  tchain_edit  libc-2.15.so        [.]
__libc_start_main
            |
            ---__libc_start_main

    99.97%     0.00%  tchain_edit  tchain_edit         [.] main
            |
            ---main
               __libc_start_main

    99.97%     0.00%  tchain_edit  tchain_edit         [.] f1
            |
            ---f1
               main
               __libc_start_main

    99.85%    87.01%  tchain_edit  tchain_edit         [.] f3
            |
            ---f3
               |
               |--99.74%-- f2
               |          f1
               |          main
               |          __libc_start_main
                --0.26%-- [...]
    99.71%     0.12%  tchain_edit  tchain_edit         [.] f2
            |
            ---f2
               f1
               main
               __libc_start_main

The callgraph from PARTIAL
# Samples: 417K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 8346980000
#

 # Children      Self  Command      Shared Object     Symbol
 # ........  ........  ...........  ................
..........................................
 #
    98.82%     0.00%  tchain_edit  libc-2.15.so      [.]
__libc_start_main
            |
            ---__libc_start_main

    98.82%     0.00%  tchain_edit  tchain_edit       [.] main
            |
            ---main
               __libc_start_main

    98.82%     0.00%  tchain_edit  tchain_edit       [.] f1
            |
            ---f1
               main
               __libc_start_main

    98.82%    98.28%  tchain_edit  tchain_edit       [.] f3
            |
            ---f3
               |
               |--0.53%-- f2
               |          f1
               |          main
               |          __libc_start_main
               |
               |--0.01%-- f1
               |          main
               |          __libc_start_main
                --99.46%-- [...]
    97.63%     0.03%  tchain_edit  tchain_edit       [.] f2
            |
            ---f2
               f1
               main
               __libc_start_main

     7.13%     0.03%  tchain_edit  [kernel.vmlinux]  [k] do_nmi
            |
            ---do_nmi
               end_repeat_nmi
               f3
               f2
               f1
               main
               __libc_start_main

---

Changes since V8:
- Same as global dump_size caculation, round_up dump_size for per-event dump_size.

 tools/perf/Documentation/perf-record.txt |  4 ++
 tools/perf/util/evsel.c                  | 67 +++++++++++++++++++++++++++++++-
 tools/perf/util/evsel.h                  |  4 ++
 tools/perf/util/parse-events.c           | 12 ++++++
 tools/perf/util/parse-events.h           |  2 +
 tools/perf/util/parse-events.l           |  2 +
 tools/perf/util/pmu.c                    |  3 +-
 7 files changed, 91 insertions(+), 3 deletions(-)
tools/perf/Documentation/perf-record.txt +4 -0
index 0d852d1..e633711 100644
@@ -52,6 +52,10 @@ OPTIONS
 	  - 'time': Disable/enable time stamping. Acceptable values are 1 for
 		    enabling time stamping. 0 for disabling time stamping.
 		    The default is 1.
+	  - 'call-graph': Disable/enable callgraph. Acceptable str are "fp" for
+			 FP mode, "dwarf" for DWARF mode, "lbr" for LBR mode and
+			 "no" for disable callgraph.
+	  - 'stack-size': user stack size for dwarf mode
 	  Note: If user explicitly sets options which conflict with the params,
 	  the value set by the params will be overridden.
 
tools/perf/util/evsel.c +65 -2

93 lines not on this page · raw message

tools/perf/util/evsel.h +4 -0
index 6a12908..09a3022 100644
@@ -40,6 +40,8 @@ struct cgroup_sel;
 enum {
 	PERF_EVSEL__CONFIG_TERM_PERIOD,
 	PERF_EVSEL__CONFIG_TERM_TIME,
+	PERF_EVSEL__CONFIG_TERM_CALLGRAPH,
+	PERF_EVSEL__CONFIG_TERM_STACK_USER,
 	PERF_EVSEL__CONFIG_TERM_MAX,
 };
 
@@ -49,6 +51,8 @@ struct perf_evsel_config_term {
 	union {
 		u64	period;
 		bool	time;
+		char	*callgraph;
+		u64	stack_user;
 	} val;
 };
 
tools/perf/util/parse-events.c +12 -0
index 828936d..773fe8b 100644
@@ -611,6 +611,12 @@ do {									   \
 			return -EINVAL;
 		}
 		break;
+	case PARSE_EVENTS__TERM_TYPE_CALLGRAPH:
+		CHECK_TYPE_VAL(STR);
+		break;
+	case PARSE_EVENTS__TERM_TYPE_STACKSIZE:
+		CHECK_TYPE_VAL(NUM);
+		break;
 	case PARSE_EVENTS__TERM_TYPE_NAME:
 		CHECK_TYPE_VAL(STR);
 		break;
@@ -662,6 +668,12 @@ do {								\
 		case PARSE_EVENTS__TERM_TYPE_TIME:
 			ADD_CONFIG_TERM(TIME, time, term->val.num);
 			break;
+		case PARSE_EVENTS__TERM_TYPE_CALLGRAPH:
+			ADD_CONFIG_TERM(CALLGRAPH, callgraph, term->val.str);
+			break;
+		case PARSE_EVENTS__TERM_TYPE_STACKSIZE:
+			ADD_CONFIG_TERM(STACK_USER, stack_user, term->val.num);
+			break;
 		default:
 			break;
 		}
tools/perf/util/parse-events.h +2 -0
index e6f9aacc..87dc9f6 100644
@@ -64,6 +64,8 @@ enum {
 	PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD,
 	PARSE_EVENTS__TERM_TYPE_BRANCH_SAMPLE_TYPE,
 	PARSE_EVENTS__TERM_TYPE_TIME,
+	PARSE_EVENTS__TERM_TYPE_CALLGRAPH,
+	PARSE_EVENTS__TERM_TYPE_STACKSIZE,
 };
 
 struct parse_events_term {
tools/perf/util/parse-events.l +2 -0
index f542750..1665497 100644
@@ -184,6 +184,8 @@ name			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_NAME); }
 period			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD); }
 branch_type		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_BRANCH_SAMPLE_TYPE); }
 time			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_TIME); }
+call-graph		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_CALLGRAPH); }
+stack-size		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_STACKSIZE); }
 ,			{ return ','; }
 "/"			{ BEGIN(INITIAL); return '/'; }
 {name_minus}		{ return str(yyscanner, PE_NAME); }
tools/perf/util/pmu.c +2 -1
index d4b0e64..239a653 100644
@@ -634,7 +634,8 @@ static char *formats_error_string(struct list_head *formats)
 {
 	struct perf_pmu_format *format;
 	char *err, *str;
-	static const char *static_terms = "config,config1,config2,name,period,branch_type,time\n";
+	static const char *static_terms = "config,config1,config2,name,period,"
+					  "branch_type,time,call-graph,stack-size\n";
 	unsigned i = 0;
 
 	if (!asprintf(&str, "valid terms:"))
Signature · 1 line
1.8.3.1
Acked-by on v9 2/3
#2 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

On Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] wrote:

SNIP

Quoted 29 lines
            |
            ---f3
               |
               |--0.53%-- f2
               |          f1
               |          main
               |          __libc_start_main
               |
               |--0.01%-- f1
               |          main
               |          __libc_start_main
                --99.46%-- [...]
    97.63%     0.03%  tchain_edit  tchain_edit       [.] f2
            |
            ---f2
               f1
               main
               __libc_start_main

     7.13%     0.03%  tchain_edit  [kernel.vmlinux]  [k] do_nmi
            |
            ---do_nmi
               end_repeat_nmi
               f3
               f2
               f1
               main
               __libc_start_main
Quoted 5 lines

---

Changes since V8:
- Same as global dump_size caculation, round_up dump_size for per-event dump_size.

thanks,
jirka

#3 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

In reply to [email protected]

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Jiri, while testing this I noticed that the message for EINVAL when using the cpu// syntax (per-event settings) is cryptic:

[root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
Error:
The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p). /bin/dmesg may provide additional information.
No CONFIG_PERF_EVENTS=y kernel support configured?

Whereas if we use -F, it is much, much clearer, telling the user exactly what is failing and what needs to be done to make it work:

[root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1 Maximum frequency rate (25000) reached.
Please use -F freq option with lower value or consider tweaking /proc/sys/kernel/perf_event_max_sample_rate. [root@zoo ~]#

Hope this is something easy to wire up, given your event parsing kung foo skillz...

;-)

- Arnaldo

#4 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 10 lines

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Some other oddity I noticed while testing:

For:

[root@zoo ~]# perf report --header-only
# ========

# captured on: Fri Aug  7 12:39:13 2015
# hostname : zoo
# os release : 4.2.0-rc5+
# perf version : 4.2.rc4.g1cd951
# arch : x86_64
# nrcpus online : 4
# nrcpus avail : 4
# cpudesc : Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz
# cpuid : GenuineIntel,6,58,9
# total memory : 8080692 kB
# cmdline : /home/acme/bin/perf record -e {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p} -a 
# event : name = cpu/cpu-cycles,call-graph=fp,time,period=10000/pp, , id = { 38537, 38538, 38539, 38540 }, type = 4, size = 112, config = 0x3c, { sample_perio
# event : name = cpu/instructions,call-graph=no,time=0,period=20000/p, , id = { 38541, 38542, 38543, 38544 }, type = 4, size = 112, config = 0xc0, { sample_pe
# HEADER_CPU_TOPOLOGY info available, use -I to display
# HEADER_NUMA_TOPOLOGY info available, use -I to display
# pmu mappings: cpu = 4, software = 1, power = 7, uncore_imc = 8, tracepoint = 2, uncore_cbox_0 = 9, uncore_cbox_1 = 10, breakpoint = 5
# group: {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p}
# ========
#

I get:

Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', Event count (approx.): 46956518

  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.] 0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] main                ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] run_builtin         ▒
     run_builtin                                                             ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 46956518
#

# Children      Self  Command          Shared Object               Symbol                                        
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155

Can you take a look? What I have, i.e. the patches from Jiri and Kan moving stuff out of the way of the python binding and this patch is at the tmp.perf/core branch in my git tree:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

- Arnaldo

Quoted 24 lines

Jiri, while testing this I noticed that the message for EINVAL when using the cpu// syntax (per-event settings) is cryptic:

[root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
Error:
The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p). /bin/dmesg may provide additional information.
No CONFIG_PERF_EVENTS=y kernel support configured?

Whereas if we use -F, it is much, much clearer, telling the user exactly what is failing and what needs to be done to make it work:

[root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1 Maximum frequency rate (25000) reached.
Please use -F freq option with lower value or consider tweaking /proc/sys/kernel/perf_event_max_sample_rate. [root@zoo ~]#

Hope this is something easy to wire up, given your event parsing kung foo skillz...

;-)

- Arnaldo

#5 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Quoted 14 lines · Arnaldo Carvalho de Melo

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 10 lines

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Some other oddity I noticed while testing:

SNIP

Quoted 33 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 46956518
#

# Children      Self  Command          Shared Object               Symbol                                        
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155
Quoted 5 lines

Can you take a look? What I have, i.e. the patches from Jiri and Kan moving stuff out of the way of the python binding and this patch is at the tmp.perf/core branch in my git tree:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

hum, tried this branch and got proper behaviour for both stdio and TUI, attaching stdio output

the record command was:

  $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls

got the events strings from your report output.. what was different in your record command?

jirka

# To display the perf.data header info, please use --header/--header-only options. #
#
# Total Lost Samples: 4
#
# Samples: 95 of event 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp' # Event count (approx.): 7487606
#

# Children      Self  Command  Shared Object       Symbol                               
# ........  ........  .......  ..................  .....................................
#
    61.66%     0.00%  ls       [kernel.vmlinux]    [k] entry_SYSCALL_64_fastpath        
                 |
                 ---entry_SYSCALL_64_fastpath
                    |          
                    |--55.19%-- open64
                    |          0x686361632e6f732e
                    |          
                    |--21.81%-- __GI___libc_write
                    |          |          
                    |          |--24.74%-- 0x6e69746c69756220
                    |          |          
                    |          |--19.20%-- 0x909632e66666964
                    |          |          
                    |          |--18.35%-- 0x657461746f6e6e61
                    |          |          
                    |          |--16.05%-- 0x1dbdc70
                    |          |          
                    |          |--11.32%-- 0
                    |          |          
                    |           --10.34%-- 0x9096f2e61746164
                    |          
                    |--6.18%-- __getdents64
                    |          
                    |--5.00%-- mmap64
                    |          |          
                    |          |--50.39%-- _dl_check_all_versions
                    |          |          
                    |           --49.61%-- _dl_map_object
                    |                     0x213988
                    |                     0
                    |          
                    |--2.72%-- mprotect
                    |          dl_main
                    |          _dl_sysdep_start
                    |          
                    |--1.26%-- _exit
                    |          __run_exit_handlers
                    |          0
                    |          
                    |--0.92%-- __GI___munmap
                     --6.92%-- [...]

    34.03%     0.00%  ls       [unknown]           [.] 0x686361632e6f732e               
                 |
                 ---0x686361632e6f732e

    34.03%     0.00%  ls       ld-2.18.so          [.] open64                           
                 |
                 ---open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] sys_open                         
                 |
                 ---sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_sys_open                      
                 |
                 ---do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_filp_open                     
                 |
                 ---do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] path_openat                      
                 |
                 ---path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] link_path_walk                   
                 |
                 ---link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] inode_permission                 
                 |
                 ---inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] __inode_permission               
                 |
                 ---__inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_interrupt               
                 |
                 ---irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] smp_irq_work_interrupt           
                 |
                 ---smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run                     
                 |
                 ---irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run_list                
                 |
                 ---irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] perf_duration_warn               
                 |
                 ---perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] printk                           
                 |
                 ---printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_default                  
                 |
                 ---vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_emit                     
                 |
                 ---vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] console_unlock                   
                 |
                 ---console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] call_console_drivers.constprop.27
                 |
                 ---call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] univ8250_console_write           
                 |
                 ---univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] uart_console_write               
                 |
                 ---uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] serial8250_console_putchar       
                 |
                 ---serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    28.96%     0.00%  ls       [kernel.vmlinux]    [k] wait_for_xmitr                   
                 |
                 ---wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    17.71%     0.00%  ls       libc-2.18.so        [.] __GI___libc_write                
                 |
                 ---__GI___libc_write
                    |          
                    |--16.81%-- 0x657461746f6e6e61
                    |          
                    |--10.79%-- 0x6e69746c69756220
                    |          
                    |--8.37%-- 0x909632e66666964
                    |          
                    |--7.00%-- 0x1dbdc70
                    |          
                    |--6.67%-- 0x96f2e68636e6562
                    |          
                    |--5.30%-- 0x2d6469646c697562
                    |          
                    |--4.93%-- 0
                    |          
                    |--4.51%-- 0x9096f2e61746164
                    |          
                    |--3.30%-- 0x6f2e7473696c7665
                     --32.32%-- [...]

    14.14%     0.41%  ls       [kernel.vmlinux]    [k] __const_udelay                   
                 |
                 ---__const_udelay
                    |          
                    |--97.12%-- wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.88%-- serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

    13.75%    13.75%  ls       [kernel.vmlinux]    [k] io_serial_in                     
                 |
                 ---io_serial_in
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] sys_write                        
                 |
                 ---sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] vfs_write                        
                 |
                 ---vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] __vfs_write                      
                 |
                 ---__vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] tty_write                        
                 |
                 ---tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] n_tty_write                      
                 |
                 ---n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    12.39%    12.39%  ls       [kernel.vmlinux]    [k] delay_tsc                        
                 |
                 ---delay_tsc
                    |          
                    |--88.16%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --11.84%-- wait_for_xmitr
                               serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     9.88%     0.00%  ls       [unknown]           [.] 0x0000000000841f0f               
                 |
                 ---0x841f0f

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] pty_write                        
                 |
                 ---pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_flip_buffer_push             
                 |
                 ---tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_schedule_flip                
                 |
                 ---tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] queue_work_on                    
                 |
                 ---queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] __queue_work                     
                 |
                 ---__queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     1.30%  ls       [kernel.vmlinux]    [k] insert_work                      
                 |
                 ---insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     8.41%     0.00%  ls       [unknown]           [.] 0000000000000000                 
                 |
                 ---0
                    |          
                    |--15.30%-- 0x632e74617473
                    |          
                    |--12.88%-- 0
                    |          
                    |--12.01%-- 0x72616863656d6974
                     --59.81%-- [...]

     7.92%     0.00%  ls       ls                  [.] 0xffffffffffc0e1cf               
                 |
                 ---0xe1cf
                    0x841f0f

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_worker                   
                 |
                 ---wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_process                  
                 |
                 ---wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] try_to_wake_up                   
                 |
                 ---try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_activate.constprop.92    
                 |
                 ---ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     5.94%     5.94%  ls       libc-2.18.so        [.] get_next_seq                     
                 |
                 ---get_next_seq
                    0xe1cf
                    0x841f0f

     5.80%     0.00%  ls       [kernel.vmlinux]    [k] __schedule                       
                 |
                 ---__schedule
                    |          
                    |--73.77%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--28.65%-- 0
                    |          |          
                    |          |--26.18%-- 0x9096f2e61746164
                    |          |          
                    |          |--24.86%-- 0x909632e66666964
                    |          |          
                    |           --20.31%-- 0x1dbdc70
                    |          
                     --26.23%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               |          
                               |--61.62%-- 0x2d6469646c697562
                               |          
                                --38.38%-- 0x6f2e7473696c7665

     5.75%     1.94%  ls       libc-2.18.so        [.] __getdents64                     
                 |
                 ---__getdents64
                    |          
                    |--33.68%-- 0x616863656d69742d
                     --66.32%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] 0x000000007f47cb44               
                 |
                 ---0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] do_group_exit                    
                 |
                 ---do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.78%  ls       [kernel.vmlinux]    [k] do_exit                          
                 |
                 ---do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     4.96%     0.00%  ls       [kernel.vmlinux]    [k] do_output_char                   
                 |
                 ---do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.59%-- 0x657461746f6e6e61
                    |          
                    |--12.51%-- 0x1dbdc70
                     --58.90%-- [...]

     4.79%     4.79%  ls       [kernel.vmlinux]    [k] native_write_msr_safe            
                 |
                 ---native_write_msr_safe
                    |          
                    |--37.79%-- __intel_pmu_disable_all
                    |          intel_pmu_disable_all
                    |          x86_pmu_disable
                    |          perf_pmu_disable
                    |          ctx_sched_out
                    |          __perf_event_task_sched_out
                    |          __schedule
                    |          |          
                    |          |--51.77%-- schedule
                    |          |          prepare_exit_to_usermode
                    |          |          syscall_return_slowpath
                    |          |          int_ret_from_sys_call
                    |          |          __GI___libc_write
                    |          |          0x2d6469646c697562
                    |          |          
                    |           --48.23%-- preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0
                    |          
                    |--32.55%-- __GI___libc_write
                    |          0x657461746f6e6e61
                    |          
                    |--29.58%-- __intel_pmu_enable_all.isra.9
                    |          intel_pmu_enable_all
                    |          x86_pmu_enable
                    |          perf_pmu_enable
                    |          |          
                    |          |--56.32%-- perf_event_context_sched_in
                    |          |          |          
                    |          |          |--100.00%-- __perf_event_task_sched_in
                    |          |          |          finish_task_switch
                    |          |          |          __schedule
                    |          |          |          preempt_schedule_common
                    |          |          |          _cond_resched
                    |          |          |          mutex_lock
                    |          |          |          n_tty_write
                    |          |          |          tty_write
                    |          |          |          __vfs_write
                    |          |          |          vfs_write
                    |          |          |          sys_write
                    |          |          |          entry_SYSCALL_64_fastpath
                    |          |          |          __GI___libc_write
                    |          |          |          0x9096f2e61746164
                    |          |           --0.00%-- [...]
                    |          |          
                    |           --43.68%-- perf_pmu_sched_task
                    |                     __perf_event_task_sched_out
                    |                     __schedule
                    |                     preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0x1dbdc70
                     --0.08%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] mutex_lock                       
                 |
                 ---mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] _cond_resched                    
                 |
                 ---_cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] preempt_schedule_common          
                 |
                 ---preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.06%     4.06%  ls       [kernel.vmlinux]    [k] io_serial_out                    
                 |
                 ---io_serial_out
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     4.04%     4.04%  ls       ls                  [.] 0x0000000000006686               
                 |
                 ---0x6686

     4.04%     0.00%  ls       ls                  [.] 0xffffffffffc06686               
                 |
                 ---0x6686

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] sys_getdents                     
                 |
                 ---sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] iterate_dir                      
                 |
                 ---iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_readdir                     
                 |
                 ---ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_htree_fill_tree             
                 |
                 ---ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] htree_dirblock_to_tree           
                 |
                 ---htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] activate_task                    
                 |
                 ---activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task                     
                 |
                 ---enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task_fair                
                 |
                 ---enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.38%     0.00%  ls       ld-2.18.so          [.] _dl_map_object                   
                 |
                 ---_dl_map_object
                    |          
                    |--45.27%-- 0x21eb10
                    |          0
                    |          
                    |--45.27%-- 0x213988
                    |          0
                    |          
                     --9.45%-- 0x21e640
                               0

     3.27%     0.00%  ls       ld-2.18.so          [.] _dl_sysdep_start                 
                 |
                 ---_dl_sysdep_start

     3.27%     0.00%  ls       ld-2.18.so          [.] dl_main                          
                 |
                 ---dl_main
                    _dl_sysdep_start

     3.08%     0.00%  ls       ld-2.18.so          [.] mmap64                           
                 |
                 ---mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap                         
                 |
                 ---sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap_pgoff                   
                 |
                 ---sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] vm_mmap_pgoff                    
                 |
                 ---vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] do_mmap_pgoff                    
                 |
                 ---do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] mmap_region                      
                 |
                 ---mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] mmput                            
                 |
                 ---mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] exit_mmap                        
                 |
                 ---exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     2.98%     0.00%  ls       [unknown]           [.] 0x657461746f6e6e61               
                 |
                 ---0x657461746f6e6e61

     2.79%     0.64%  ls       [kernel.vmlinux]    [k] finish_task_switch               
                 |
                 ---finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.65%-- 0x9096f2e61746164
                    |          
                    |--27.20%-- 0x909632e66666964
                     --44.14%-- [...]

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] int_ret_from_sys_call            
                 |
                 ---int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] syscall_return_slowpath          
                 |
                 ---syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     1.18%  ls       [kernel.vmlinux]    [k] prepare_exit_to_usermode         
                 |
                 ---prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.43%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_out      
                 |
                 ---__perf_event_task_sched_out
                    __schedule
                    |          
                    |--61.42%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--58.52%-- 0
                    |          |          
                    |           --41.48%-- 0x1dbdc70
                    |          
                     --38.58%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               0x2d6469646c697562

     2.30%     0.00%  ls       ls                  [.] 0xffffffffffc06dcb               
                 |
                 ---0x6dcb
                    0
                    |          
                    |--56.03%-- 0x632e74617473
                    |          
                     --43.97%-- 0x72616863656d6974

     2.26%     1.63%  ls       ld-2.18.so          [.] do_lookup_x                      
                 |
                 ---do_lookup_x

     2.16%     1.54%  ls       [kernel.vmlinux]    [k] nmi                              
                 |
                 ---nmi
                    |          
                    |--63.15%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--29.07%-- do_lookup_x
                    |          
                    |--5.52%-- end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--1.10%-- nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--0.63%-- perf_event_nmi_handler
                    |          nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                     --0.52%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     2.15%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_in       
                 |
                 ---__perf_event_task_sched_in
                    finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--37.19%-- 0x9096f2e61746164
                    |          
                    |--35.31%-- 0x909632e66666964
                     --27.50%-- [...]

     2.06%     0.00%  ls       [kernel.vmlinux]    [k] tty_put_char                     
                 |
                 ---tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     2.06%     2.06%  ls       [kernel.vmlinux]    [k] update_rq_runnable_avg           
                 |
                 ---update_rq_runnable_avg
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     1.98%     1.98%  ls       libc-2.18.so        [.] __strcoll_l                      
                 |
                 ---__strcoll_l
                    0xe1cf
                    0x841f0f

     1.97%     1.35%  ls       [kernel.vmlinux]    [k] perf_pmu_sched_task              
                 |
                 ---perf_pmu_sched_task
                    |          
                    |--68.52%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x909632e66666964
                    |          
                     --31.48%-- __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.95%     1.95%  ls       ls                  [.] 0x000000000000e270               
                 |
                 ---0xe270
                    0x841f0f

     1.95%     0.00%  ls       ls                  [.] 0xffffffffffc0e270               
                 |
                 ---0xe270
                    0x841f0f

     1.94%     0.00%  ls       [unknown]           [.] 0x616863656d69742d               
                 |
                 ---0x616863656d69742d

     1.93%     0.00%  ls       [kernel.vmlinux]    [k] unmap_vmas                       
                 |
                 ---unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.93%  ls       [kernel.vmlinux]    [k] unmap_single_vma                 
                 |
                 ---unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.00%  ls       [ext4]              [k] ext4fs_dirhash                   
                 |
                 ---ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.93%     1.93%  ls       [ext4]              [k] str2hashbuf_signed               
                 |
                 ---str2hashbuf_signed
                    ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.91%     0.00%  ls       [unknown]           [.] 0x6e69746c69756220               
                 |
                 ---0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_wakeup                   
                 |
                 ---ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_curr               
                 |
                 ---check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_wakeup             
                 |
                 ---check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     1.91%  ls       [kernel.vmlinux]    [k] set_next_buddy                   
                 |
                 ---set_next_buddy
                    check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.89%     1.89%  ls       [ext4]              [k] ext4_htree_store_dirent          
                 |
                 ---ext4_htree_store_dirent
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.85%     1.85%  ls       libc-2.18.so        [.] __strcmp_sse2                    
                 |
                 ---__strcmp_sse2

     1.82%     0.00%  ls       [unknown]           [.] 0x6f74207367756220               
                 |
                 ---0x6f74207367756220

     1.82%     1.82%  ls       libc-2.18.so        [.] strlen                           
                 |
                 ---strlen
                    0x6f74207367756220

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] ctx_sched_out                    
                 |
                 ---ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_disable                 
                 |
                 ---perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_disable                  
                 |
                 ---x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_disable_all            
                 |
                 ---intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_disable_all          
                 |
                 ---__intel_pmu_disable_all
                    intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.76%     0.00%  ls       [unknown]           [.] 0x00401f0fc3f30678               
                 |
                 ---0x401f0fc3f30678

     1.76%     1.76%  ls       libc-2.18.so        [.] _dl_addr                         
                 |
                 ---_dl_addr
                    0x401f0fc3f30678

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc06438               
                 |
                 ---0x6438

     1.72%     1.72%  ls       ls                  [.] 0x0000000000005b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc05b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.70%     1.70%  ls       libpthread-2.18.so  [.] frame_dummy                      
                 |
                 ---frame_dummy

     1.68%     0.00%  ls       ld-2.18.so          [.] mprotect                         
                 |
                 ---mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] sys_mprotect                     
                 |
                 ---sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] mprotect_fixup                   
                 |
                 ---mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] split_vma                        
                 |
                 ---split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] __split_vma.isra.34              
                 |
                 ---__split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     1.68%  ls       [kernel.vmlinux]    [k] vma_interval_tree_remove         
                 |
                 ---vma_interval_tree_remove
                    __split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.59%     1.59%  ls       ld-2.18.so          [.] _dl_relocate_object              
                 |
                 ---_dl_relocate_object
                    dl_main
                    _dl_sysdep_start

     1.58%     0.86%  ls       [kernel.vmlinux]    [k] enqueue_entity                   
                 |
                 ---enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--54.22%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --45.78%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     1.55%     0.00%  ls       ld-2.18.so          [.] _dl_check_all_versions           
                 |
                 ---_dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_mmap                  
                 |
                 ---perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_aux                   
                 |
                 ---perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     1.55%  ls       [kernel.vmlinux]    [k] perf_event_aux_ctx               
                 |
                 ---perf_event_aux_ctx
                    perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb57988               
                 |
                 ---0x213988
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62b10               
                 |
                 ---0x21eb10
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] memset                           
                 |
                 ---memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] page_fault                       
                 |
                 ---page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] do_page_fault                    
                 |
                 ---do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] __do_page_fault                  
                 |
                 ---__do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] handle_mm_fault                  
                 |
                 ---handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] put_page                         
                 |
                 ---put_page
                    handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] vma_link                         
                 |
                 ---vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] vma_interval_tree_insert         
                 |
                 ---vma_interval_tree_insert
                    vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.52%     0.00%  ls       [kernel.vmlinux]    [k] schedule                         
                 |
                 ---schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--61.62%-- 0x2d6469646c697562
                    |          
                     --38.38%-- 0x6f2e7473696c7665

     1.48%     0.00%  ls       [unknown]           [.] 0x0909632e66666964               
                 |
                 ---0x909632e66666964

     1.44%     1.44%  ls       [kernel.vmlinux]    [k] paranoid_entry                   
                 |
                 ---paranoid_entry
                    __const_udelay
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_enable                  
                 |
                 ---perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_enable                   
                 |
                 ---x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_enable_all             
                 |
                 ---intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_enable_all.isra.9    
                 |
                 ---__intel_pmu_enable_all.isra.9
                    intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] wq_worker_waking_up              
                 |
                 ---wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.42%     1.42%  ls       [kernel.vmlinux]    [k] kthread_data                     
                 |
                 ---kthread_data
                    wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.31%     0.04%  ls       [kernel.vmlinux]    [k] do_nmi                           
                 |
                 ---do_nmi
                    |          
                    |--52.08%-- end_repeat_nmi
                    |          |          
                    |          |--46.71%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--41.63%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --11.66%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --47.92%-- nmi
                               do_lookup_x

     1.29%     0.00%  ls       [unknown]           [.] 0x0000632e74617473               
                 |
                 ---0x632e74617473

     1.29%     1.29%  ls       ls                  [.] 0x0000000000005a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.29%     0.00%  ls       ls                  [.] 0xffffffffffc05a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.27%     0.00%  ls       [kernel.vmlinux]    [k] default_do_nmi                   
                 |
                 ---default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.27%     0.63%  ls       [kernel.vmlinux]    [k] nmi_handle                       
                 |
                 ---nmi_handle
                    default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.24%     0.00%  ls       [unknown]           [.] 0x0000000001dbdc70               
                 |
                 ---0x1dbdc70

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] task_work_run                    
                 |
                 ---task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] ____fput                         
                 |
                 ---____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] __fput                           
                 |
                 ---__fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     1.23%  ls       [kernel.vmlinux]    [k] locks_remove_file                
                 |
                 ---locks_remove_file
                    __fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.18%     0.00%  ls       [unknown]           [.] 0x096f2e68636e6562               
                 |
                 ---0x96f2e68636e6562

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_finish_mmu                   
                 |
                 ---tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_flush_mmu_free               
                 |
                 ---tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] free_pages_and_swap_cache        
                 |
                 ---free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     1.10%  ls       [kernel.vmlinux]    [k] release_pages                    
                 |
                 ---release_pages
                    free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.08%     1.08%  ls       ls                  [.] 0x0000000000006b20               
                 |
                 ---0x6b20
                    0
                    0

     1.08%     0.00%  ls       ls                  [.] 0xffffffffffc06b20               
                 |
                 ---0x6b20
                    0
                    0

     1.01%     0.00%  ls       [unknown]           [.] 0x72616863656d6974               
                 |
                 ---0x72616863656d6974

     1.01%     1.01%  ls       ls                  [.] __ctype_get_mb_cur_max@plt       
                 |
                 ---__ctype_get_mb_cur_max@plt
                    0x6dcb
                    0
                    0x72616863656d6974

     1.01%     1.01%  ls       [kernel.vmlinux]    [k] unmap_page_range                 
                 |
                 ---unmap_page_range
                    unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     0.94%     0.00%  ls       [unknown]           [.] 0x2d6469646c697562               
                 |
                 ---0x2d6469646c697562

     0.89%     0.89%  ls       [kernel.vmlinux]    [k] get_pwq.isra.18                  
                 |
                 ---get_pwq.isra.18
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] end_repeat_nmi                   
                 |
                 ---end_repeat_nmi
                    |          
                    |--39.76%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--35.44%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --24.80%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_context_sched_in      
                 |
                 ---perf_event_context_sched_in
                    |          
                    |--100.00%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          0x9096f2e61746164
                     --0.00%-- [...]

     0.80%     0.00%  ls       [unknown]           [.] 0x09096f2e61746164               
                 |
                 ---0x9096f2e61746164

     0.78%     0.00%  ls       libc-2.18.so        [.] __run_exit_handlers              
                 |
                 ---__run_exit_handlers
                    0

     0.78%     0.00%  ls       libc-2.18.so        [.] _exit                            
                 |
                 ---_exit
                    __run_exit_handlers
                    0

     0.72%     0.72%  ls       [kernel.vmlinux]    [k] account_entity_enqueue           
                 |
                 ---account_entity_enqueue
                    enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x909632e66666964

     0.62%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_nmi_handler           
                 |
                 ---perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--51.70%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--46.08%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.22%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.60%     0.60%  ls       [kernel.vmlinux]    [k] perf_sample_event_took           
                 |
                 ---perf_sample_event_took
                    perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--52.87%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --47.13%-- __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     0.58%     0.00%  ls       [unknown]           [.] 0x6f2e7473696c7665               
                 |
                 ---0x6f2e7473696c7665

     0.58%     0.00%  ls       [kernel.vmlinux]    [k] pick_next_task_fair              
                 |
                 ---pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.58%     0.58%  ls       [kernel.vmlinux]    [k] __enqueue_entity                 
                 |
                 ---__enqueue_entity
                    pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.57%     0.00%  ls       libc-2.18.so        [.] __GI___munmap                    
                 |
                 ---__GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] sys_munmap                       
                 |
                 ---sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] vm_munmap                        
                 |
                 ---vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] do_munmap                        
                 |
                 ---do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] remove_vma                       
                 |
                 ---remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.57%  ls       [kernel.vmlinux]    [k] kmem_cache_free                  
                 |
                 ---kmem_cache_free
                    remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] msg_print_text                   
                 |
                 ---msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_prefix                     
                 |
                 ---print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_time.part.12               
                 |
                 ---print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] sprintf                          
                 |
                 ---sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] vsnprintf                        
                 |
                 ---vsnprintf
                    sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62640               
                 |
                 ---0x21e640
                    0

     0.32%     0.32%  ls       ld-2.18.so          [.] _dl_setup_hash                   
                 |
                 ---_dl_setup_hash
                    _dl_map_object
                    0x21e640
                    0

     0.29%     0.00%  ls       [unknown]           [k] 0x00007f9abb498a77               
                 |
                 ---0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] return_from_execve               
                 |
                 ---return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] sys_execve                       
                 |
                 ---sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] do_execveat_common.isra.32       
                 |
                 ---do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] search_binary_handler            
                 |
                 ---search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] load_elf_binary                  
                 |
                 ---load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] setup_new_exec                   
                 |
                 ---setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] apic_timer_interrupt             
                 |
                 ---apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] smp_apic_timer_interrupt         
                 |
                 ---smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] local_apic_timer_interrupt       
                 |
                 ---local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] hrtimer_interrupt                
                 |
                 ---hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] __hrtimer_run_queues             
                 |
                 ---__hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_timer                 
                 |
                 ---tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_handle.isra.17        
                 |
                 ---tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] update_process_times             
                 |
                 ---update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] scheduler_tick                   
                 |
                 ---scheduler_tick
                    update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.08%     0.08%  ls       [kernel.vmlinux]    [k] native_sched_clock               
                 |
                 ---native_sched_clock
                    0x7f9abb498a77

     0.00%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_exec                  
                 |
                 ---perf_event_exec
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

# Samples: 158 of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 3762761
#

# Children      Self  Command  Shared Object     Symbol                            
# ........  ........  .......  ................  ..................................
#
     5.87%     5.87%  ls       [unknown]         [.] 0x00007fb353d5cf7e            

     4.84%     4.84%  ls       [kernel.vmlinux]  [k] delay_tsc                     

     3.77%     3.77%  ls       [kernel.vmlinux]  [k] pick_next_task_fair           

     3.11%     3.11%  ls       [kernel.vmlinux]  [k] unmap_page_range              

     2.94%     2.94%  ls       [kernel.vmlinux]  [k] n_tty_write                   

     2.48%     2.48%  ls       [unknown]         [.] 0x000000000040669a            

     2.40%     2.40%  ls       [unknown]         [.] 0x00007fb353d5cd1a            

     2.37%     2.37%  ls       [unknown]         [.] 0x00007fb353d5cca4            

     2.37%     2.37%  ls       [unknown]         [.] 0x00000000004066ca            

     2.25%     2.25%  ls       [unknown]         [.] 0x00007fb353d5db87            

     2.03%     2.03%  ls       [unknown]         [.] 0x00007fb353d5ccaa            

     1.98%     1.98%  ls       [unknown]         [.] 0x00007fb353d5cc9c            

     1.97%     1.97%  ls       [unknown]         [.] 0x0000000000405b2a            

     1.89%     1.89%  ls       [unknown]         [.] 0x00007fb3546d43e0            

     1.83%     1.83%  ls       [unknown]         [.] 0x00007fb353d5cef6            

     1.75%     1.75%  ls       [kernel.vmlinux]  [k] ring_buffer_record_is_on      

     1.69%     1.69%  ls       [unknown]         [.] 0x00007fb353d5cc3a            

     1.63%     1.63%  ls       [unknown]         [.] 0x0000000000405611            

     1.47%     1.47%  ls       [kernel.vmlinux]  [k] __fput                        

     1.44%     1.44%  ls       [kernel.vmlinux]  [k] release_pages                 

     1.37%     1.37%  ls       [kernel.vmlinux]  [k] perf_pmu_sched_task           

     1.34%     1.34%  ls       [unknown]         [.] 0x00007fb353d54fed            

     1.33%     1.33%  ls       [kernel.vmlinux]  [k] __memcpy                      

     1.33%     1.33%  ls       [unknown]         [.] 0x00007fb353d5df34            

     1.31%     1.31%  ls       [unknown]         [.] 0x00007fb353d5ccdf            

     1.29%     1.29%  ls       [unknown]         [.] 0x0000000000406a19            

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] page_remove_rmap              

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] do_set_pte                    

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] __audit_syscall_exit          

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] enqueue_entity                

     1.28%     1.28%  ls       [ext4]            [k] ext4fs_dirhash                

     1.26%     1.26%  ls       [kernel.vmlinux]  [k] cpuacct_charge                

     1.24%     1.24%  ls       [kernel.vmlinux]  [k] sched_clock                   

     1.24%     1.24%  ls       [unknown]         [.] 0x00007fb353d4bca5            

     1.22%     1.22%  ls       [unknown]         [.] 0x00007fb353dfb6ac            

     1.21%     1.21%  ls       [ext4]            [k] str2hashbuf_signed            

     1.21%     1.21%  ls       [unknown]         [.] 0x00007fb3546c766a            

     1.20%     1.20%  ls       [ext4]            [k] __ext4_check_dir_entry        

     1.13%     1.13%  ls       [kernel.vmlinux]  [k] x86_pmu_disable               

     1.12%     1.12%  ls       [unknown]         [.] 0x00007fb3546c5571            

     1.11%     1.11%  ls       [kernel.vmlinux]  [k] vm_normal_page                

     1.09%     1.09%  ls       [unknown]         [.] 0x00007fb3546cc246            

     1.06%     1.06%  ls       [unknown]         [.] 0x00007fb3546cc21e            

     1.03%     1.03%  ls       [unknown]         [.] 0x000000000040e6a0            

     1.01%     1.01%  ls       [unknown]         [.] 0x00007fb3546c5ce3            

     0.88%     0.88%  ls       [unknown]         [.] 0x00007fb3546c7814            

     0.88%     0.88%  ls       [kernel.vmlinux]  [k] mark_page_accessed            

     0.87%     0.87%  ls       [unknown]         [.] 0x00007fb3546c7960            

     0.78%     0.78%  ls       [unknown]         [.] 0x000000000040e7ea            

     0.77%     0.77%  ls       [kernel.vmlinux]  [k] security_mmap_file            

     0.66%     0.66%  ls       [unknown]         [.] 0x000000000040f3c5            

     0.65%     0.65%  ls       [kernel.vmlinux]  [k] enqueue_task_fair             

     0.63%     0.63%  ls       [kernel.vmlinux]  [k] unlock_page                   

     0.61%     0.61%  ls       [kernel.vmlinux]  [k] place_entity                  

     0.57%     0.57%  ls       [unknown]         [.] 0x000000000040e808            

     0.56%     0.56%  ls       [kernel.vmlinux]  [k] perf_event_aux_ctx            

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] __rb_insert_augmented         

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] update_min_vruntime           

     0.50%     0.50%  ls       [kernel.vmlinux]  [k] update_curr                   

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] vma_interval_tree_insert      

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] io_serial_in                  

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] check_cfs_rq_runtime          

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] tracing_record_cmdline        

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] __dec_zone_page_state         

     0.42%     0.42%  ls       [kernel.vmlinux]  [k] try_to_wake_up                

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] insert_work                   

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] put_prev_entity               

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] avc_has_perm_noaudit          

     0.38%     0.38%  ls       [kernel.vmlinux]  [k] __list_del_entry              

     0.38%     0.38%  ls       [unknown]         [.] 0x0000000000405852            

     0.36%     0.36%  ls       [kernel.vmlinux]  [k] __intel_pmu_disable_all       

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] ttwu_do_activate.constprop.92 

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] mem_cgroup_begin_page_stat    

     0.34%     0.34%  ls       [kernel.vmlinux]  [k] perf_ctx_unlock               

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405b46            

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405811            

     0.31%     0.31%  ls       [unknown]         [.] 0x00007fb3546c54fa            

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] rb_next                       

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] tracing_is_on                 

     0.30%     0.30%  ls       [unknown]         [.] 0x00007fb3546c553d            

     0.29%     0.29%  ls       [unknown]         [.] 0x00007fb3546cb597            

     0.20%     0.20%  ls       [kernel.vmlinux]  [k] format_decode                 

     0.18%     0.18%  ls       [kernel.vmlinux]  [k] __audit_syscall_entry         

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] arch_get_unmapped_area_topdown

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] fsnotify                      

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] intel_bts_enable_local        

     0.15%     0.15%  ls       [kernel.vmlinux]  [k] selinux_inode_permission      

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] wait_for_xmitr                

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] __const_udelay                

     0.11%     0.11%  ls       [kernel.vmlinux]  [k] find_vma                      

     0.11%     0.11%  ls       [unknown]         [.] 0x00007fb3546d18ae            

     0.10%     0.10%  ls       [unknown]         [.] 0x00007fb3546cb18e            

     0.03%     0.03%  ls       [kernel.vmlinux]  [k] native_sched_clock            

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] trigger_load_balance          

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] do_nmi                        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_event_nmi_handler        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_sample_event_took        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] nmi                           

#
# (For a higher level overview, try: perf report --sort comm,dso) #

#6 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Em Sat, Aug 08, 2015 at 06:45:39PM +0200, Jiri Olsa escreveu:

Quoted 57 lines · Arnaldo Carvalho de Melo
Quoted 14 lines

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 10 lines

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Some other oddity I noticed while testing:

SNIP

Quoted 33 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 46956518
#

# Children      Self  Command          Shared Object               Symbol                                        
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155
Quoted 5 lines

Can you take a look? What I have, i.e. the patches from Jiri and Kan moving stuff out of the way of the python binding and this patch is at the tmp.perf/core branch in my git tree:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

Quoted 7 lines

hum, tried this branch and got proper behaviour for both stdio and TUI, attaching stdio output

the record command was:

  $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls

got the events strings from your report output.. what was different in your record command?

Was in my message, here it is:

# cmdline : /home/acme/bin/perf record -e {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p}
-a

I'll try this again later.

- Arnaldo

#7 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

In reply to Jiri Olsa

Em Sat, Aug 08, 2015 at 06:45:39PM +0200, Jiri Olsa escreveu:

Quoted 51 lines · Arnaldo Carvalho de Melo
Quoted 14 lines

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 10 lines

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Some other oddity I noticed while testing:

SNIP

Quoted 33 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 46956518
#

# Children      Self  Command          Shared Object               Symbol                                        
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155
Quoted 7 lines
Quoted 5 lines

Can you take a look? What I have, i.e. the patches from Jiri and Kan moving stuff out of the way of the python binding and this patch is at the tmp.perf/core branch in my git tree:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

hum, tried this branch and got proper behaviour for both stdio and TUI,

Well, from what he explained to me, the proper behaviour was the one I got, i.e. one event has call-graph=fp, and thus the perf_event_attr fields for callchains will be set up as always, and the other, with call-graph=no, will not ask the kernel for callchains, thus saving space, but report _will_ show callchains for it, using the callchains obtained for the first event.

So, "call-graph=no" doesn't mean you don't want callchains for a particular events _if_ there is another event in the group for which callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)

I think we should use "call-graph=ref" to mean that no callchains should be requested to the kernel infrastructure but that when doing the report, use callchains available in some other event (perhaps would be good to specify which one), while "call-graph=no" really means "no", i.e. no callchains asked from the kernel for this event, and _no_ callchains to appear on report.

- Arnaldo

Quoted 4,022 lines

attaching stdio output

the record command was:

  $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls

got the events strings from your report output.. what was different in your record command?

jirka

# To display the perf.data header info, please use --header/--header-only options. #
#
# Total Lost Samples: 4
#
# Samples: 95 of event 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp' # Event count (approx.): 7487606
#

# Children      Self  Command  Shared Object       Symbol                               
# ........  ........  .......  ..................  .....................................
#
    61.66%     0.00%  ls       [kernel.vmlinux]    [k] entry_SYSCALL_64_fastpath        
                 |
                 ---entry_SYSCALL_64_fastpath
                    |          
                    |--55.19%-- open64
                    |          0x686361632e6f732e
                    |          
                    |--21.81%-- __GI___libc_write
                    |          |          
                    |          |--24.74%-- 0x6e69746c69756220
                    |          |          
                    |          |--19.20%-- 0x909632e66666964
                    |          |          
                    |          |--18.35%-- 0x657461746f6e6e61
                    |          |          
                    |          |--16.05%-- 0x1dbdc70
                    |          |          
                    |          |--11.32%-- 0
                    |          |          
                    |           --10.34%-- 0x9096f2e61746164
                    |          
                    |--6.18%-- __getdents64
                    |          
                    |--5.00%-- mmap64
                    |          |          
                    |          |--50.39%-- _dl_check_all_versions
                    |          |          
                    |           --49.61%-- _dl_map_object
                    |                     0x213988
                    |                     0
                    |          
                    |--2.72%-- mprotect
                    |          dl_main
                    |          _dl_sysdep_start
                    |          
                    |--1.26%-- _exit
                    |          __run_exit_handlers
                    |          0
                    |          
                    |--0.92%-- __GI___munmap
                     --6.92%-- [...]

    34.03%     0.00%  ls       [unknown]           [.] 0x686361632e6f732e               
                 |
                 ---0x686361632e6f732e

    34.03%     0.00%  ls       ld-2.18.so          [.] open64                           
                 |
                 ---open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] sys_open                         
                 |
                 ---sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_sys_open                      
                 |
                 ---do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_filp_open                     
                 |
                 ---do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] path_openat                      
                 |
                 ---path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] link_path_walk                   
                 |
                 ---link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] inode_permission                 
                 |
                 ---inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] __inode_permission               
                 |
                 ---__inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_interrupt               
                 |
                 ---irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] smp_irq_work_interrupt           
                 |
                 ---smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run                     
                 |
                 ---irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run_list                
                 |
                 ---irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] perf_duration_warn               
                 |
                 ---perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] printk                           
                 |
                 ---printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_default                  
                 |
                 ---vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_emit                     
                 |
                 ---vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] console_unlock                   
                 |
                 ---console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] call_console_drivers.constprop.27
                 |
                 ---call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] univ8250_console_write           
                 |
                 ---univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] uart_console_write               
                 |
                 ---uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] serial8250_console_putchar       
                 |
                 ---serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    28.96%     0.00%  ls       [kernel.vmlinux]    [k] wait_for_xmitr                   
                 |
                 ---wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    17.71%     0.00%  ls       libc-2.18.so        [.] __GI___libc_write                
                 |
                 ---__GI___libc_write
                    |          
                    |--16.81%-- 0x657461746f6e6e61
                    |          
                    |--10.79%-- 0x6e69746c69756220
                    |          
                    |--8.37%-- 0x909632e66666964
                    |          
                    |--7.00%-- 0x1dbdc70
                    |          
                    |--6.67%-- 0x96f2e68636e6562
                    |          
                    |--5.30%-- 0x2d6469646c697562
                    |          
                    |--4.93%-- 0
                    |          
                    |--4.51%-- 0x9096f2e61746164
                    |          
                    |--3.30%-- 0x6f2e7473696c7665
                     --32.32%-- [...]

    14.14%     0.41%  ls       [kernel.vmlinux]    [k] __const_udelay                   
                 |
                 ---__const_udelay
                    |          
                    |--97.12%-- wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.88%-- serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

    13.75%    13.75%  ls       [kernel.vmlinux]    [k] io_serial_in                     
                 |
                 ---io_serial_in
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] sys_write                        
                 |
                 ---sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] vfs_write                        
                 |
                 ---vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] __vfs_write                      
                 |
                 ---__vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] tty_write                        
                 |
                 ---tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] n_tty_write                      
                 |
                 ---n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    12.39%    12.39%  ls       [kernel.vmlinux]    [k] delay_tsc                        
                 |
                 ---delay_tsc
                    |          
                    |--88.16%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --11.84%-- wait_for_xmitr
                               serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     9.88%     0.00%  ls       [unknown]           [.] 0x0000000000841f0f               
                 |
                 ---0x841f0f

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] pty_write                        
                 |
                 ---pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_flip_buffer_push             
                 |
                 ---tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_schedule_flip                
                 |
                 ---tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] queue_work_on                    
                 |
                 ---queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] __queue_work                     
                 |
                 ---__queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     1.30%  ls       [kernel.vmlinux]    [k] insert_work                      
                 |
                 ---insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     8.41%     0.00%  ls       [unknown]           [.] 0000000000000000                 
                 |
                 ---0
                    |          
                    |--15.30%-- 0x632e74617473
                    |          
                    |--12.88%-- 0
                    |          
                    |--12.01%-- 0x72616863656d6974
                     --59.81%-- [...]

     7.92%     0.00%  ls       ls                  [.] 0xffffffffffc0e1cf               
                 |
                 ---0xe1cf
                    0x841f0f

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_worker                   
                 |
                 ---wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_process                  
                 |
                 ---wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] try_to_wake_up                   
                 |
                 ---try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_activate.constprop.92    
                 |
                 ---ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     5.94%     5.94%  ls       libc-2.18.so        [.] get_next_seq                     
                 |
                 ---get_next_seq
                    0xe1cf
                    0x841f0f

     5.80%     0.00%  ls       [kernel.vmlinux]    [k] __schedule                       
                 |
                 ---__schedule
                    |          
                    |--73.77%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--28.65%-- 0
                    |          |          
                    |          |--26.18%-- 0x9096f2e61746164
                    |          |          
                    |          |--24.86%-- 0x909632e66666964
                    |          |          
                    |           --20.31%-- 0x1dbdc70
                    |          
                     --26.23%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               |          
                               |--61.62%-- 0x2d6469646c697562
                               |          
                                --38.38%-- 0x6f2e7473696c7665

     5.75%     1.94%  ls       libc-2.18.so        [.] __getdents64                     
                 |
                 ---__getdents64
                    |          
                    |--33.68%-- 0x616863656d69742d
                     --66.32%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] 0x000000007f47cb44               
                 |
                 ---0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] do_group_exit                    
                 |
                 ---do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.78%  ls       [kernel.vmlinux]    [k] do_exit                          
                 |
                 ---do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     4.96%     0.00%  ls       [kernel.vmlinux]    [k] do_output_char                   
                 |
                 ---do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.59%-- 0x657461746f6e6e61
                    |          
                    |--12.51%-- 0x1dbdc70
                     --58.90%-- [...]

     4.79%     4.79%  ls       [kernel.vmlinux]    [k] native_write_msr_safe            
                 |
                 ---native_write_msr_safe
                    |          
                    |--37.79%-- __intel_pmu_disable_all
                    |          intel_pmu_disable_all
                    |          x86_pmu_disable
                    |          perf_pmu_disable
                    |          ctx_sched_out
                    |          __perf_event_task_sched_out
                    |          __schedule
                    |          |          
                    |          |--51.77%-- schedule
                    |          |          prepare_exit_to_usermode
                    |          |          syscall_return_slowpath
                    |          |          int_ret_from_sys_call
                    |          |          __GI___libc_write
                    |          |          0x2d6469646c697562
                    |          |          
                    |           --48.23%-- preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0
                    |          
                    |--32.55%-- __GI___libc_write
                    |          0x657461746f6e6e61
                    |          
                    |--29.58%-- __intel_pmu_enable_all.isra.9
                    |          intel_pmu_enable_all
                    |          x86_pmu_enable
                    |          perf_pmu_enable
                    |          |          
                    |          |--56.32%-- perf_event_context_sched_in
                    |          |          |          
                    |          |          |--100.00%-- __perf_event_task_sched_in
                    |          |          |          finish_task_switch
                    |          |          |          __schedule
                    |          |          |          preempt_schedule_common
                    |          |          |          _cond_resched
                    |          |          |          mutex_lock
                    |          |          |          n_tty_write
                    |          |          |          tty_write
                    |          |          |          __vfs_write
                    |          |          |          vfs_write
                    |          |          |          sys_write
                    |          |          |          entry_SYSCALL_64_fastpath
                    |          |          |          __GI___libc_write
                    |          |          |          0x9096f2e61746164
                    |          |           --0.00%-- [...]
                    |          |          
                    |           --43.68%-- perf_pmu_sched_task
                    |                     __perf_event_task_sched_out
                    |                     __schedule
                    |                     preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0x1dbdc70
                     --0.08%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] mutex_lock                       
                 |
                 ---mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] _cond_resched                    
                 |
                 ---_cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] preempt_schedule_common          
                 |
                 ---preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.06%     4.06%  ls       [kernel.vmlinux]    [k] io_serial_out                    
                 |
                 ---io_serial_out
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     4.04%     4.04%  ls       ls                  [.] 0x0000000000006686               
                 |
                 ---0x6686

     4.04%     0.00%  ls       ls                  [.] 0xffffffffffc06686               
                 |
                 ---0x6686

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] sys_getdents                     
                 |
                 ---sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] iterate_dir                      
                 |
                 ---iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_readdir                     
                 |
                 ---ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_htree_fill_tree             
                 |
                 ---ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] htree_dirblock_to_tree           
                 |
                 ---htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] activate_task                    
                 |
                 ---activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task                     
                 |
                 ---enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task_fair                
                 |
                 ---enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.38%     0.00%  ls       ld-2.18.so          [.] _dl_map_object                   
                 |
                 ---_dl_map_object
                    |          
                    |--45.27%-- 0x21eb10
                    |          0
                    |          
                    |--45.27%-- 0x213988
                    |          0
                    |          
                     --9.45%-- 0x21e640
                               0

     3.27%     0.00%  ls       ld-2.18.so          [.] _dl_sysdep_start                 
                 |
                 ---_dl_sysdep_start

     3.27%     0.00%  ls       ld-2.18.so          [.] dl_main                          
                 |
                 ---dl_main
                    _dl_sysdep_start

     3.08%     0.00%  ls       ld-2.18.so          [.] mmap64                           
                 |
                 ---mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap                         
                 |
                 ---sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap_pgoff                   
                 |
                 ---sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] vm_mmap_pgoff                    
                 |
                 ---vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] do_mmap_pgoff                    
                 |
                 ---do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] mmap_region                      
                 |
                 ---mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] mmput                            
                 |
                 ---mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] exit_mmap                        
                 |
                 ---exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     2.98%     0.00%  ls       [unknown]           [.] 0x657461746f6e6e61               
                 |
                 ---0x657461746f6e6e61

     2.79%     0.64%  ls       [kernel.vmlinux]    [k] finish_task_switch               
                 |
                 ---finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.65%-- 0x9096f2e61746164
                    |          
                    |--27.20%-- 0x909632e66666964
                     --44.14%-- [...]

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] int_ret_from_sys_call            
                 |
                 ---int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] syscall_return_slowpath          
                 |
                 ---syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     1.18%  ls       [kernel.vmlinux]    [k] prepare_exit_to_usermode         
                 |
                 ---prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.43%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_out      
                 |
                 ---__perf_event_task_sched_out
                    __schedule
                    |          
                    |--61.42%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--58.52%-- 0
                    |          |          
                    |           --41.48%-- 0x1dbdc70
                    |          
                     --38.58%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               0x2d6469646c697562

     2.30%     0.00%  ls       ls                  [.] 0xffffffffffc06dcb               
                 |
                 ---0x6dcb
                    0
                    |          
                    |--56.03%-- 0x632e74617473
                    |          
                     --43.97%-- 0x72616863656d6974

     2.26%     1.63%  ls       ld-2.18.so          [.] do_lookup_x                      
                 |
                 ---do_lookup_x

     2.16%     1.54%  ls       [kernel.vmlinux]    [k] nmi                              
                 |
                 ---nmi
                    |          
                    |--63.15%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--29.07%-- do_lookup_x
                    |          
                    |--5.52%-- end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--1.10%-- nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--0.63%-- perf_event_nmi_handler
                    |          nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                     --0.52%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     2.15%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_in       
                 |
                 ---__perf_event_task_sched_in
                    finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--37.19%-- 0x9096f2e61746164
                    |          
                    |--35.31%-- 0x909632e66666964
                     --27.50%-- [...]

     2.06%     0.00%  ls       [kernel.vmlinux]    [k] tty_put_char                     
                 |
                 ---tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     2.06%     2.06%  ls       [kernel.vmlinux]    [k] update_rq_runnable_avg           
                 |
                 ---update_rq_runnable_avg
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     1.98%     1.98%  ls       libc-2.18.so        [.] __strcoll_l                      
                 |
                 ---__strcoll_l
                    0xe1cf
                    0x841f0f

     1.97%     1.35%  ls       [kernel.vmlinux]    [k] perf_pmu_sched_task              
                 |
                 ---perf_pmu_sched_task
                    |          
                    |--68.52%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x909632e66666964
                    |          
                     --31.48%-- __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.95%     1.95%  ls       ls                  [.] 0x000000000000e270               
                 |
                 ---0xe270
                    0x841f0f

     1.95%     0.00%  ls       ls                  [.] 0xffffffffffc0e270               
                 |
                 ---0xe270
                    0x841f0f

     1.94%     0.00%  ls       [unknown]           [.] 0x616863656d69742d               
                 |
                 ---0x616863656d69742d

     1.93%     0.00%  ls       [kernel.vmlinux]    [k] unmap_vmas                       
                 |
                 ---unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.93%  ls       [kernel.vmlinux]    [k] unmap_single_vma                 
                 |
                 ---unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.00%  ls       [ext4]              [k] ext4fs_dirhash                   
                 |
                 ---ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.93%     1.93%  ls       [ext4]              [k] str2hashbuf_signed               
                 |
                 ---str2hashbuf_signed
                    ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.91%     0.00%  ls       [unknown]           [.] 0x6e69746c69756220               
                 |
                 ---0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_wakeup                   
                 |
                 ---ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_curr               
                 |
                 ---check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_wakeup             
                 |
                 ---check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     1.91%  ls       [kernel.vmlinux]    [k] set_next_buddy                   
                 |
                 ---set_next_buddy
                    check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.89%     1.89%  ls       [ext4]              [k] ext4_htree_store_dirent          
                 |
                 ---ext4_htree_store_dirent
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.85%     1.85%  ls       libc-2.18.so        [.] __strcmp_sse2                    
                 |
                 ---__strcmp_sse2

     1.82%     0.00%  ls       [unknown]           [.] 0x6f74207367756220               
                 |
                 ---0x6f74207367756220

     1.82%     1.82%  ls       libc-2.18.so        [.] strlen                           
                 |
                 ---strlen
                    0x6f74207367756220

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] ctx_sched_out                    
                 |
                 ---ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_disable                 
                 |
                 ---perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_disable                  
                 |
                 ---x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_disable_all            
                 |
                 ---intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_disable_all          
                 |
                 ---__intel_pmu_disable_all
                    intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.76%     0.00%  ls       [unknown]           [.] 0x00401f0fc3f30678               
                 |
                 ---0x401f0fc3f30678

     1.76%     1.76%  ls       libc-2.18.so        [.] _dl_addr                         
                 |
                 ---_dl_addr
                    0x401f0fc3f30678

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc06438               
                 |
                 ---0x6438

     1.72%     1.72%  ls       ls                  [.] 0x0000000000005b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc05b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.70%     1.70%  ls       libpthread-2.18.so  [.] frame_dummy                      
                 |
                 ---frame_dummy

     1.68%     0.00%  ls       ld-2.18.so          [.] mprotect                         
                 |
                 ---mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] sys_mprotect                     
                 |
                 ---sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] mprotect_fixup                   
                 |
                 ---mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] split_vma                        
                 |
                 ---split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] __split_vma.isra.34              
                 |
                 ---__split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     1.68%  ls       [kernel.vmlinux]    [k] vma_interval_tree_remove         
                 |
                 ---vma_interval_tree_remove
                    __split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.59%     1.59%  ls       ld-2.18.so          [.] _dl_relocate_object              
                 |
                 ---_dl_relocate_object
                    dl_main
                    _dl_sysdep_start

     1.58%     0.86%  ls       [kernel.vmlinux]    [k] enqueue_entity                   
                 |
                 ---enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--54.22%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --45.78%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     1.55%     0.00%  ls       ld-2.18.so          [.] _dl_check_all_versions           
                 |
                 ---_dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_mmap                  
                 |
                 ---perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_aux                   
                 |
                 ---perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     1.55%  ls       [kernel.vmlinux]    [k] perf_event_aux_ctx               
                 |
                 ---perf_event_aux_ctx
                    perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb57988               
                 |
                 ---0x213988
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62b10               
                 |
                 ---0x21eb10
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] memset                           
                 |
                 ---memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] page_fault                       
                 |
                 ---page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] do_page_fault                    
                 |
                 ---do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] __do_page_fault                  
                 |
                 ---__do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] handle_mm_fault                  
                 |
                 ---handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] put_page                         
                 |
                 ---put_page
                    handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] vma_link                         
                 |
                 ---vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] vma_interval_tree_insert         
                 |
                 ---vma_interval_tree_insert
                    vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.52%     0.00%  ls       [kernel.vmlinux]    [k] schedule                         
                 |
                 ---schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--61.62%-- 0x2d6469646c697562
                    |          
                     --38.38%-- 0x6f2e7473696c7665

     1.48%     0.00%  ls       [unknown]           [.] 0x0909632e66666964               
                 |
                 ---0x909632e66666964

     1.44%     1.44%  ls       [kernel.vmlinux]    [k] paranoid_entry                   
                 |
                 ---paranoid_entry
                    __const_udelay
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_enable                  
                 |
                 ---perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_enable                   
                 |
                 ---x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_enable_all             
                 |
                 ---intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_enable_all.isra.9    
                 |
                 ---__intel_pmu_enable_all.isra.9
                    intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] wq_worker_waking_up              
                 |
                 ---wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.42%     1.42%  ls       [kernel.vmlinux]    [k] kthread_data                     
                 |
                 ---kthread_data
                    wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.31%     0.04%  ls       [kernel.vmlinux]    [k] do_nmi                           
                 |
                 ---do_nmi
                    |          
                    |--52.08%-- end_repeat_nmi
                    |          |          
                    |          |--46.71%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--41.63%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --11.66%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --47.92%-- nmi
                               do_lookup_x

     1.29%     0.00%  ls       [unknown]           [.] 0x0000632e74617473               
                 |
                 ---0x632e74617473

     1.29%     1.29%  ls       ls                  [.] 0x0000000000005a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.29%     0.00%  ls       ls                  [.] 0xffffffffffc05a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.27%     0.00%  ls       [kernel.vmlinux]    [k] default_do_nmi                   
                 |
                 ---default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.27%     0.63%  ls       [kernel.vmlinux]    [k] nmi_handle                       
                 |
                 ---nmi_handle
                    default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.24%     0.00%  ls       [unknown]           [.] 0x0000000001dbdc70               
                 |
                 ---0x1dbdc70

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] task_work_run                    
                 |
                 ---task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] ____fput                         
                 |
                 ---____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] __fput                           
                 |
                 ---__fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     1.23%  ls       [kernel.vmlinux]    [k] locks_remove_file                
                 |
                 ---locks_remove_file
                    __fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.18%     0.00%  ls       [unknown]           [.] 0x096f2e68636e6562               
                 |
                 ---0x96f2e68636e6562

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_finish_mmu                   
                 |
                 ---tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_flush_mmu_free               
                 |
                 ---tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] free_pages_and_swap_cache        
                 |
                 ---free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     1.10%  ls       [kernel.vmlinux]    [k] release_pages                    
                 |
                 ---release_pages
                    free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.08%     1.08%  ls       ls                  [.] 0x0000000000006b20               
                 |
                 ---0x6b20
                    0
                    0

     1.08%     0.00%  ls       ls                  [.] 0xffffffffffc06b20               
                 |
                 ---0x6b20
                    0
                    0

     1.01%     0.00%  ls       [unknown]           [.] 0x72616863656d6974               
                 |
                 ---0x72616863656d6974

     1.01%     1.01%  ls       ls                  [.] __ctype_get_mb_cur_max@plt       
                 |
                 ---__ctype_get_mb_cur_max@plt
                    0x6dcb
                    0
                    0x72616863656d6974

     1.01%     1.01%  ls       [kernel.vmlinux]    [k] unmap_page_range                 
                 |
                 ---unmap_page_range
                    unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     0.94%     0.00%  ls       [unknown]           [.] 0x2d6469646c697562               
                 |
                 ---0x2d6469646c697562

     0.89%     0.89%  ls       [kernel.vmlinux]    [k] get_pwq.isra.18                  
                 |
                 ---get_pwq.isra.18
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] end_repeat_nmi                   
                 |
                 ---end_repeat_nmi
                    |          
                    |--39.76%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--35.44%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --24.80%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_context_sched_in      
                 |
                 ---perf_event_context_sched_in
                    |          
                    |--100.00%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          0x9096f2e61746164
                     --0.00%-- [...]

     0.80%     0.00%  ls       [unknown]           [.] 0x09096f2e61746164               
                 |
                 ---0x9096f2e61746164

     0.78%     0.00%  ls       libc-2.18.so        [.] __run_exit_handlers              
                 |
                 ---__run_exit_handlers
                    0

     0.78%     0.00%  ls       libc-2.18.so        [.] _exit                            
                 |
                 ---_exit
                    __run_exit_handlers
                    0

     0.72%     0.72%  ls       [kernel.vmlinux]    [k] account_entity_enqueue           
                 |
                 ---account_entity_enqueue
                    enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x909632e66666964

     0.62%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_nmi_handler           
                 |
                 ---perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--51.70%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--46.08%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.22%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.60%     0.60%  ls       [kernel.vmlinux]    [k] perf_sample_event_took           
                 |
                 ---perf_sample_event_took
                    perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--52.87%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --47.13%-- __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     0.58%     0.00%  ls       [unknown]           [.] 0x6f2e7473696c7665               
                 |
                 ---0x6f2e7473696c7665

     0.58%     0.00%  ls       [kernel.vmlinux]    [k] pick_next_task_fair              
                 |
                 ---pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.58%     0.58%  ls       [kernel.vmlinux]    [k] __enqueue_entity                 
                 |
                 ---__enqueue_entity
                    pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.57%     0.00%  ls       libc-2.18.so        [.] __GI___munmap                    
                 |
                 ---__GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] sys_munmap                       
                 |
                 ---sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] vm_munmap                        
                 |
                 ---vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] do_munmap                        
                 |
                 ---do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] remove_vma                       
                 |
                 ---remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.57%  ls       [kernel.vmlinux]    [k] kmem_cache_free                  
                 |
                 ---kmem_cache_free
                    remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] msg_print_text                   
                 |
                 ---msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_prefix                     
                 |
                 ---print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_time.part.12               
                 |
                 ---print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] sprintf                          
                 |
                 ---sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] vsnprintf                        
                 |
                 ---vsnprintf
                    sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62640               
                 |
                 ---0x21e640
                    0

     0.32%     0.32%  ls       ld-2.18.so          [.] _dl_setup_hash                   
                 |
                 ---_dl_setup_hash
                    _dl_map_object
                    0x21e640
                    0

     0.29%     0.00%  ls       [unknown]           [k] 0x00007f9abb498a77               
                 |
                 ---0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] return_from_execve               
                 |
                 ---return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] sys_execve                       
                 |
                 ---sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] do_execveat_common.isra.32       
                 |
                 ---do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] search_binary_handler            
                 |
                 ---search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] load_elf_binary                  
                 |
                 ---load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] setup_new_exec                   
                 |
                 ---setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] apic_timer_interrupt             
                 |
                 ---apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] smp_apic_timer_interrupt         
                 |
                 ---smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] local_apic_timer_interrupt       
                 |
                 ---local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] hrtimer_interrupt                
                 |
                 ---hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] __hrtimer_run_queues             
                 |
                 ---__hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_timer                 
                 |
                 ---tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_handle.isra.17        
                 |
                 ---tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] update_process_times             
                 |
                 ---update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] scheduler_tick                   
                 |
                 ---scheduler_tick
                    update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.08%     0.08%  ls       [kernel.vmlinux]    [k] native_sched_clock               
                 |
                 ---native_sched_clock
                    0x7f9abb498a77

     0.00%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_exec                  
                 |
                 ---perf_event_exec
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

# Samples: 158 of event 'cpu/instructions,call-graph=no,time=0,period=20000/p' # Event count (approx.): 3762761
#

# Children      Self  Command  Shared Object     Symbol                            
# ........  ........  .......  ................  ..................................
#
     5.87%     5.87%  ls       [unknown]         [.] 0x00007fb353d5cf7e            

     4.84%     4.84%  ls       [kernel.vmlinux]  [k] delay_tsc                     

     3.77%     3.77%  ls       [kernel.vmlinux]  [k] pick_next_task_fair           

     3.11%     3.11%  ls       [kernel.vmlinux]  [k] unmap_page_range              

     2.94%     2.94%  ls       [kernel.vmlinux]  [k] n_tty_write                   

     2.48%     2.48%  ls       [unknown]         [.] 0x000000000040669a            

     2.40%     2.40%  ls       [unknown]         [.] 0x00007fb353d5cd1a            

     2.37%     2.37%  ls       [unknown]         [.] 0x00007fb353d5cca4            

     2.37%     2.37%  ls       [unknown]         [.] 0x00000000004066ca            

     2.25%     2.25%  ls       [unknown]         [.] 0x00007fb353d5db87            

     2.03%     2.03%  ls       [unknown]         [.] 0x00007fb353d5ccaa            

     1.98%     1.98%  ls       [unknown]         [.] 0x00007fb353d5cc9c            

     1.97%     1.97%  ls       [unknown]         [.] 0x0000000000405b2a            

     1.89%     1.89%  ls       [unknown]         [.] 0x00007fb3546d43e0            

     1.83%     1.83%  ls       [unknown]         [.] 0x00007fb353d5cef6            

     1.75%     1.75%  ls       [kernel.vmlinux]  [k] ring_buffer_record_is_on      

     1.69%     1.69%  ls       [unknown]         [.] 0x00007fb353d5cc3a            

     1.63%     1.63%  ls       [unknown]         [.] 0x0000000000405611            

     1.47%     1.47%  ls       [kernel.vmlinux]  [k] __fput                        

     1.44%     1.44%  ls       [kernel.vmlinux]  [k] release_pages                 

     1.37%     1.37%  ls       [kernel.vmlinux]  [k] perf_pmu_sched_task           

     1.34%     1.34%  ls       [unknown]         [.] 0x00007fb353d54fed            

     1.33%     1.33%  ls       [kernel.vmlinux]  [k] __memcpy                      

     1.33%     1.33%  ls       [unknown]         [.] 0x00007fb353d5df34            

     1.31%     1.31%  ls       [unknown]         [.] 0x00007fb353d5ccdf            

     1.29%     1.29%  ls       [unknown]         [.] 0x0000000000406a19            

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] page_remove_rmap              

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] do_set_pte                    

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] __audit_syscall_exit          

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] enqueue_entity                

     1.28%     1.28%  ls       [ext4]            [k] ext4fs_dirhash                

     1.26%     1.26%  ls       [kernel.vmlinux]  [k] cpuacct_charge                

     1.24%     1.24%  ls       [kernel.vmlinux]  [k] sched_clock                   

     1.24%     1.24%  ls       [unknown]         [.] 0x00007fb353d4bca5            

     1.22%     1.22%  ls       [unknown]         [.] 0x00007fb353dfb6ac            

     1.21%     1.21%  ls       [ext4]            [k] str2hashbuf_signed            

     1.21%     1.21%  ls       [unknown]         [.] 0x00007fb3546c766a            

     1.20%     1.20%  ls       [ext4]            [k] __ext4_check_dir_entry        

     1.13%     1.13%  ls       [kernel.vmlinux]  [k] x86_pmu_disable               

     1.12%     1.12%  ls       [unknown]         [.] 0x00007fb3546c5571            

     1.11%     1.11%  ls       [kernel.vmlinux]  [k] vm_normal_page                

     1.09%     1.09%  ls       [unknown]         [.] 0x00007fb3546cc246            

     1.06%     1.06%  ls       [unknown]         [.] 0x00007fb3546cc21e            

     1.03%     1.03%  ls       [unknown]         [.] 0x000000000040e6a0            

     1.01%     1.01%  ls       [unknown]         [.] 0x00007fb3546c5ce3            

     0.88%     0.88%  ls       [unknown]         [.] 0x00007fb3546c7814            

     0.88%     0.88%  ls       [kernel.vmlinux]  [k] mark_page_accessed            

     0.87%     0.87%  ls       [unknown]         [.] 0x00007fb3546c7960            

     0.78%     0.78%  ls       [unknown]         [.] 0x000000000040e7ea            

     0.77%     0.77%  ls       [kernel.vmlinux]  [k] security_mmap_file            

     0.66%     0.66%  ls       [unknown]         [.] 0x000000000040f3c5            

     0.65%     0.65%  ls       [kernel.vmlinux]  [k] enqueue_task_fair             

     0.63%     0.63%  ls       [kernel.vmlinux]  [k] unlock_page                   

     0.61%     0.61%  ls       [kernel.vmlinux]  [k] place_entity                  

     0.57%     0.57%  ls       [unknown]         [.] 0x000000000040e808            

     0.56%     0.56%  ls       [kernel.vmlinux]  [k] perf_event_aux_ctx            

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] __rb_insert_augmented         

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] update_min_vruntime           

     0.50%     0.50%  ls       [kernel.vmlinux]  [k] update_curr                   

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] vma_interval_tree_insert      

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] io_serial_in                  

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] check_cfs_rq_runtime          

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] tracing_record_cmdline        

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] __dec_zone_page_state         

     0.42%     0.42%  ls       [kernel.vmlinux]  [k] try_to_wake_up                

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] insert_work                   

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] put_prev_entity               

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] avc_has_perm_noaudit          

     0.38%     0.38%  ls       [kernel.vmlinux]  [k] __list_del_entry              

     0.38%     0.38%  ls       [unknown]         [.] 0x0000000000405852            

     0.36%     0.36%  ls       [kernel.vmlinux]  [k] __intel_pmu_disable_all       

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] ttwu_do_activate.constprop.92 

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] mem_cgroup_begin_page_stat    

     0.34%     0.34%  ls       [kernel.vmlinux]  [k] perf_ctx_unlock               

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405b46            

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405811            

     0.31%     0.31%  ls       [unknown]         [.] 0x00007fb3546c54fa            

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] rb_next                       

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] tracing_is_on                 

     0.30%     0.30%  ls       [unknown]         [.] 0x00007fb3546c553d            

     0.29%     0.29%  ls       [unknown]         [.] 0x00007fb3546cb597            

     0.20%     0.20%  ls       [kernel.vmlinux]  [k] format_decode                 

     0.18%     0.18%  ls       [kernel.vmlinux]  [k] __audit_syscall_entry         

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] arch_get_unmapped_area_topdown

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] fsnotify                      

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] intel_bts_enable_local        

     0.15%     0.15%  ls       [kernel.vmlinux]  [k] selinux_inode_permission      

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] wait_for_xmitr                

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] __const_udelay                

     0.11%     0.11%  ls       [kernel.vmlinux]  [k] find_vma                      

     0.11%     0.11%  ls       [unknown]         [.] 0x00007fb3546d18ae            

     0.10%     0.10%  ls       [unknown]         [.] 0x00007fb3546cb18e            

     0.03%     0.03%  ls       [kernel.vmlinux]  [k] native_sched_clock            

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] trigger_load_balance          

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] do_nmi                        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_event_nmi_handler        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_sample_event_took        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] nmi                           

#
# (For a higher level overview, try: perf report --sort comm,dso) #

author
#8 lore raw

RE: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

In reply to Arnaldo Carvalho de Melo
Quoted 67 lines

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 5 lines

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 4 lines

From: Kan Liang <[email protected]> Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -

H".

Quoted 5 lines
Quoted 5 lines

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Some other oddity I noticed while testing:

For:

[root@zoo ~]# perf report --header-only
# ========

# captured on: Fri Aug  7 12:39:13 2015
# hostname : zoo
# os release : 4.2.0-rc5+
# perf version : 4.2.rc4.g1cd951
# arch : x86_64
# nrcpus online : 4
# nrcpus avail : 4
# cpudesc : Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz # cpuid :
GenuineIntel,6,58,9 # total memory : 8080692 kB # cmdline :
/home/acme/bin/perf record -e {cpu/cpu-cycles,call-
graph=fp,time,period=10000/pp,cpu/instructions,call-
graph=no,time=0,period=20000/p} -a # event : name = cpu/cpu-cycles,call-
graph=fp,time,period=10000/pp, , id = { 38537, 38538, 38539, 38540 }, type
= 4, size = 112, config = 0x3c, { sample_perio # event : name =
cpu/instructions,call-graph=no,time=0,period=20000/p, , id = { 38541,
38542, 38543, 38544 }, type = 4, size = 112, config = 0xc0, { sample_pe #
HEADER_CPU_TOPOLOGY info available, use -I to display #
HEADER_NUMA_TOPOLOGY info available, use -I to display # pmu
mappings: cpu = 4, software = 1, power = 7, uncore_imc = 8, tracepoint = 2,
uncore_cbox_0 = 9, uncore_cbox_1 = 10, breakpoint = 5 # group:
{cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-
graph=no,time=0,period=20000/p}
# ========
#

I get:

Samples: 2K of event 'cpu/instructions,call- graph=no,time=0,period=20000/p', Event count (approx.): 46956518

  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒
(quoted diff) +0 -3
-   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] main                ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] run_builtin         ▒
     run_builtin                                                             ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒
Quoted 3 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

That's the design.
For sampling multiple events, it may not be needed to collect callgraphs for all of them. Because the sample sites are usually nearby. It's enough to collect the callgraphs on a reference event. For other events, it can still show callgraphs according to the callgraphs on a reference event.

Thanks,
Kan

Quoted 42 lines

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-
graph=no,time=0,period=20000/p'
# Event count (approx.): 46956518
#

# Children      Self  Command          Shared Object               Symbol
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.]
0xad5e258d4c544155
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.]
__libc_start_main
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155

Can you take a look? What I have, i.e. the patches from Jiri and Kan moving stuff out of the way of the python binding and this patch is at the tmp.perf/core branch in my git tree:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

- Arnaldo

Quoted 26 lines
Quoted 4 lines

Jiri, while testing this I noticed that the message for EINVAL when using the cpu// syntax (per-event settings) is cryptic:

[root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-

graph=fp,time,period=100000/p' ls

Quoted 2 lines

Error:
The sys_perf_event_open() syscall returned with 22 (Invalid argument)

for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p).

Quoted 18 lines

/bin/dmesg may provide additional information. No CONFIG_PERF_EVENTS=y kernel support configured?

Whereas if we use -F, it is much, much clearer, telling the user exactly what is failing and what needs to be done to make it work:

[root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1 Maximum frequency rate (25000) reached.
Please use -F freq option with lower value or consider tweaking /proc/sys/kernel/perf_event_max_sample_rate. [root@zoo ~]#

Hope this is something easy to wire up, given your event parsing kung foo skillz...

;-)

- Arnaldo

���{.n�+�������+%�����ݶ��w��{.n�+����{��G�����{ay�ʇڙ�,j��f���h��������z_�(�階�ݢj"���m������G������?����&���~��iO��z��v�^�m���� ������?�I�

#9 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Em Mon, Aug 10, 2015 at 12:56:04PM +0000, Liang, Kan escreveu:

Quoted 6 lines
Quoted 6 lines

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
[root@zoo ~]# perf report --header-only

# cmdline :
/home/acme/bin/perf record -e {cpu/cpu-cycles,call-
graph=fp,time,period=10000/pp,cpu/instructions,call-
graph=no,time=0,period=20000/p} -a

<SNIP>

Quoted 12 lines
Quoted 11 lines

I get:

Samples: 2K of event 'cpu/instructions,call- graph=no,time=0,period=20000/p', Event count (approx.): 46956518

  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
Quoted 8 lines
Quoted 2 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

That's the design.
For sampling multiple events, it may not be needed to collect callgraphs for all of them. Because the sample sites are usually nearby. It's enough to collect the callgraphs on a reference event. For other events, it can still show callgraphs according to the callgraphs on a reference event.

So, "call-graph=no" doesn't mean you don't want callchains for a particular events _if_ there is another event in the group for which callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)

I think we should use "call-graph=ref" to mean that no callchains should be requested to the kernel infrastructure for that particular event, but that when doing the report, use callchains available in some other event (perhaps would be good to specify which one), while "call-graph=no" really means "no", i.e. no callchains asked from the kernel for this event, and _no_ callchains to appear on report.

If "ref" is used and no callchains are available anywhere, that is a bug as well, i.e. I asked for callchains up to a event to be used, by getting that info from another event, but no event has callchains:
error.

- Arnaldo

author
#10 lore raw

RE: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Quoted 40 lines

<SNIP>

Quoted 9 lines
Quoted 9 lines

I get:

Samples: 2K of event 'cpu/instructions,call- graph=no,time=0,period=20000/p', Event count (approx.): 46956518

  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒

- 67.56% 0.00% qemu-system-x86 libc-2.20.so [.] __libc_start_main

▒

Quoted 11 lines
Quoted 5 lines
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

That's the design.
For sampling multiple events, it may not be needed to collect callgraphs for all of them. Because the sample sites are usually nearby. It's enough to collect the callgraphs on a reference event. For other events, it can still show callgraphs according to the callgraphs on

a reference event.

So, "call-graph=no" doesn't mean you don't want callchains for a particular events _if_ there is another event in the group for which callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)

I think we should use "call-graph=ref" to mean that no callchains should be requested to the kernel infrastructure for that particular event, but that when doing the report, use callchains available in some other event (perhaps would be good to specify which one), while "call-graph=no" really means "no", i.e. no callchains asked from the kernel for this event, and _no_ callchains to appear on report.

Quoted 5 lines

If "ref" is used and no callchains are available anywhere, that is a bug as well, i.e. I asked for callchains up to a event to be used, by getting that info from another event, but no event has callchains:
error.

If we use " call-graph=ref", it means "ref" is a new callchain mode. But it's not. I think the "ref" thing should only impact the perf report.
So we may introduce a new option "--show-callchain-ref" for that purpose. If it applied, the available callchain information from other event will be printed for "call-graph=no" event.
If not, no callchain information is printed for "call-graph=no" event. The default is no print.
Is it OK?

Thanks,
Kan ���{.n�+�������+%�����ݶ��w��{.n�+����{��G�����{ay�ʇڙ�,j��f���h��������z_�(�階�ݢj"���m������G������?����&���~��iO��z��v�^�m���� ������?�I�

#11 lore raw

Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support

Em Mon, Aug 10, 2015 at 06:57:20PM +0000, Liang, Kan escreveu:

Quoted 37 lines
Quoted 36 lines

<SNIP>

Quoted 12 lines
Quoted 11 lines

I get:

Samples: 2K of event 'cpu/instructions,call- graph=no,time=0,period=20000/p', Event count (approx.): 46956518

  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
Quoted 8 lines
Quoted 2 lines

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get callchains? How come?

That's the design.
For sampling multiple events, it may not be needed to collect callgraphs for all of them. Because the sample sites are usually nearby. It's enough to collect the callgraphs on a reference event. For other events, it can still show callgraphs according to the callgraphs on

a reference event.

So, "call-graph=no" doesn't mean you don't want callchains for a particular events _if_ there is another event in the group for which callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)

I think we should use "call-graph=ref" to mean that no callchains should be requested to the kernel infrastructure for that particular event, but that when doing the report, use callchains available in some other event (perhaps would be good to specify which one), while "call-graph=no" really means "no", i.e. no callchains asked from the kernel for this event, and _no_ callchains to appear on report.

Quoted 7 lines
Quoted 4 lines

If "ref" is used and no callchains are available anywhere, that is a bug as well, i.e. I asked for callchains up to a event to be used, by getting that info from another event, but no event has callchains:
error.

If we use " call-graph=ref", it means "ref" is a new callchain mode. But it's not. I think the "ref" thing should only impact the perf report.

I don't have much of a problem with that, but using "ref" to make the intention, i.e. use reference callchains, documented, clear, makes sense to me.

I.e. when you ask for two events, one with callchains and the other without it doesn't necessarily means we want callchains appearing on the ones we have not enabled them.

Quoted 3 lines

So we may introduce a new option "--show-callchain-ref" for that purpose. If it applied, the available callchain information from other event will be printed for "call-graph=no" event.

Ok, if the user explicitely asked for "--show-callchain-ref", then he/she will not get confused seeing callchains for an event with "call-graph=no".

Ah, probably --show-ref-call-graph should be better, to keep it consistent with all the other options dealing with call-graph stuff.

Quoted 2 lines

If not, no callchain information is printed for "call-graph=no" event. The default is no print.

Agreed, I think this almost completely reduces the possible source of confusion.

Quoted 1 line

Is it OK?

Ok.

One possible improvement to your proposal: When showing callchains in reference mode, make that extra explicit by adding some marker on the side of the event name.

I.e. right now we will see callchains, when this is with another event with callchains:

Samples: 24 of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', Event count (approx.): 480000

  Overhead  Command  Shared Object     Symbol
    12.50%  usleep   libc-2.20.so      [.] _dl_addr

My suggestion is to have something like:

Samples: 24 of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', ref cg, Event count (approx.): 480000

  Overhead  Command  Shared Object     Symbol
    12.50%  usleep   libc-2.20.so      [.] _dl_addr

See that ", ref cg"?

But would be just to remove the confusion of seeing, on the same screen, "call-graph=no" when one _sees_ call graphs.

- Arnaldo

#12 lore raw

PATCH 1/1 perf tools: Unset perf_event_attr::freq when period term is set

In reply to Arnaldo Carvalho de Melo
Quoted 23 lines · Arnaldo Carvalho de Melo

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, [email protected] escreveu:

Quoted 9 lines

From: Kan Liang <[email protected]>
Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

The test command for FULL callgraph and time support.
"perf record -e '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}' --call-graph fp --time"

Jiri, while testing this I noticed that the message for EINVAL when using the cpu// syntax (per-event settings) is cryptic:

[root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
Error:
The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p). /bin/dmesg may provide additional information.
No CONFIG_PERF_EVENTS=y kernel support configured?

Whereas if we use -F, it is much, much clearer, telling the user exactly what is failing and what needs to be done to make it work:

Quoted 8 lines

[root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1 Maximum frequency rate (25000) reached.
Please use -F freq option with lower value or consider tweaking /proc/sys/kernel/perf_event_max_sample_rate. [root@zoo ~]#

Hope this is something easy to wire up, given your event parsing kung foo skillz...

my kungu foo found there was actually another issue ;-)

we did not clear up attr->freq bit.. so the sample_period was handled as sample_freq value.. please check the patch below

anyfoo, now period setup can raise sky high (which is correct) and once it meets the God it shows:

[jolsa@krava perf]$ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=1000000000000000000000000/' ls event syntax error: '..time,period=1000000000000000000000000/'
\___ parser error

I'll check if we could add some default error message when value crosses the type maximum.. which now falls to parser error

jirka

---
We need to unset 'perf_event_attr::freq' bit (default 1) when 'period' term is specified within event definition like:

-e 'cpu/cpu-cycles,call-graph=fp,time,period=100000'

otherwise it will handle the period value as frequency (and fail if it crossed the maximum allowed frequency value).

---

tools/perf/util/evsel.c +1 -0
index d902cad4ce10..4c779d23b1d7 100644
@@ -623,6 +623,7 @@ static void apply_config_terms(struct perf_evsel *evsel,
 		switch (term->type) {
 		case PERF_EVSEL__CONFIG_TERM_PERIOD:
 			attr->sample_period = term->val.period;
+			attr->freq = 0;
 			break;
 		case PERF_EVSEL__CONFIG_TERM_TIME:
 			if (term->val.time)
#13 lore raw

[tip:perf/core] perf tools: Unset perf_event_attr:: freq when period term is set

perf tools: Unset perf_event_attr::freq when period term is set
Commit-ID:  ab35a7d0ee59a36c9c567defe43c1adb72e9240c
Gitweb:     http://git.kernel.org/tip/ab35a7d0ee59a36c9c567defe43c1adb72e9240c
Author:     Jiri Olsa <[email protected]>
AuthorDate: Sat, 8 Aug 2015 19:12:10 +0200
Committer:  Arnaldo Carvalho de Melo <[email protected]>
CommitDate: Mon, 10 Aug 2015 11:58:05 -0300

perf tools: Unset perf_event_attr::freq when period term is set

We need to unset 'perf_event_attr::freq' bit (default 1) when 'period' term is specified within event definition like:

-e 'cpu/cpu-cycles,call-graph=fp,time,period=100000'

otherwise it will handle the period value as frequency (and fail if it crossed the maximum allowed frequency value).

---
 tools/perf/util/evsel.c | 1 +
 1 file changed, 1 insertion(+)
tools/perf/util/evsel.c +1 -0
index f572f46..a59710f 100644
@@ -598,6 +598,7 @@ static void apply_config_terms(struct perf_evsel *evsel)
 		switch (term->type) {
 		case PERF_EVSEL__CONFIG_TERM_PERIOD:
 			attr->sample_period = term->val.period;
+			attr->freq = 0;
 			break;
 		case PERF_EVSEL__CONFIG_TERM_TIME:
 			if (term->val.time)
#14 lore raw

RFC PATCH v9 3/3 perf,tests: Add tests to callgraph and time parse

In reply to [email protected]

From: Kan Liang <[email protected]>

Add tests in tests/parse-events.c to check call-graph and time option

---
 tools/perf/tests/parse-events.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
tools/perf/tests/parse-events.c +38 -0
index f65bb89..9b6b2b63 100644
@@ -479,6 +479,39 @@ static int test__checkevent_pmu_name(struct perf_evlist *evlist)
 	return 0;
 }
 
+static int test__checkevent_pmu_partial_time_callgraph(struct perf_evlist *evlist)
+{
+	struct perf_evsel *evsel = perf_evlist__first(evlist);
+
+	/* cpu/config=1,call-graph=fp,time,period=100000/ */
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->nr_entries);
+	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->attr.type);
+	TEST_ASSERT_VAL("wrong config",  1 == evsel->attr.config);
+	/*
+	 * The period, time and callgraph value gets configured
+	 * within perf_evlist__config,
+	 * while this test executes only parse events method.
+	 */
+	TEST_ASSERT_VAL("wrong period",     0 == evsel->attr.sample_period);
+	TEST_ASSERT_VAL("wrong callgraph",  !(PERF_SAMPLE_CALLCHAIN & evsel->attr.sample_type));
+	TEST_ASSERT_VAL("wrong time",  !(PERF_SAMPLE_TIME & evsel->attr.sample_type));
+
+	/* cpu/config=2,call-graph=no,time=0,period=2000/ */
+	evsel = perf_evsel__next(evsel);
+	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->attr.type);
+	TEST_ASSERT_VAL("wrong config",  2 == evsel->attr.config);
+	/*
+	 * The period, time and callgraph value gets configured
+	 * within perf_evlist__config,
+	 * while this test executes only parse events method.
+	 */
+	TEST_ASSERT_VAL("wrong period",     0 == evsel->attr.sample_period);
+	TEST_ASSERT_VAL("wrong callgraph",  !(PERF_SAMPLE_CALLCHAIN & evsel->attr.sample_type));
+	TEST_ASSERT_VAL("wrong time",  !(PERF_SAMPLE_TIME & evsel->attr.sample_type));
+
+	return 0;
+}
+
 static int test__checkevent_pmu_events(struct perf_evlist *evlist)
 {
 	struct perf_evsel *evsel = perf_evlist__first(evlist);
@@ -1555,6 +1588,11 @@ static struct evlist_test test__events_pmu[] = {
 		.check = test__checkevent_pmu_name,
 		.id    = 1,
 	},
+	{
+		.name  = "cpu/config=1,call-graph=fp,time,period=100000/,cpu/config=2,call-graph=no,time=0,period=2000/",
+		.check = test__checkevent_pmu_partial_time_callgraph,
+		.id    = 2,
+	},
 };
 
 struct terms_test {
Signature · 1 line
1.8.3.1
#16 lore raw

PATCH 1/1 perf tools: Move perf_counts struct and functions into separate object

In reply to [email protected]
Quoted 3 lines · On Thu, Aug 06, 2015 at 03:44:52PM -0400, [email protected] wrote:

From: Kan Liang <[email protected]>

Move callchain option parse related code to util.c

little nore about the reason would be nice ;-)

looks ok, but the python test is still failing, the reason is the perf_counts struct objects and functions bindings to util/stat.c object

Arnaldo,
I separated the 'struct perf_counts' into util/counts.c, because I recall you did not want them in evsel.c, please check attached patch.. it makes the python test pass again (on top of Kan's change).

thanks,
jirka

---
Moving perf_counts struct and functions into separate object, so we could remove stat.c object dependency from python build.

It makes the python code to be built properly, because now it fails to load due to missing stat-shadow.c object dependency.

---

tools/perf/builtin-stat.c +1 -0
index a054ddc0b2a0..7aa039bd379a 100644
@@ -58,6 +58,7 @@
 #include "util/cpumap.h"
 #include "util/thread.h"
 #include "util/thread_map.h"
+#include "util/counts.h"
 
 #include <stdlib.h>
 #include <sys/prctl.h>
tools/perf/util/Build +1 -0
index a1e5168dc1fb..4ca481cd38e6 100644
@@ -67,6 +67,7 @@ libperf-y += target.o
 libperf-y += rblist.o
 libperf-y += intlist.o
 libperf-y += vdso.o
+libperf-y += counts.o
 libperf-y += stat.o
 libperf-y += stat-shadow.o
 libperf-y += record.o
tools/perf/util/counts.c new file +52 -0

52 lines not on this page · raw message

tools/perf/util/counts.h new file +37 -0
new file mode 100644
index 000000000000..34d8baaf558a
@@ -0,0 +1,37 @@
+#ifndef __PERF_COUNTS_H
+#define __PERF_COUNTS_H
+
+#include "xyarray.h"
+
+struct perf_counts_values {
+	union {
+		struct {
+			u64 val;
+			u64 ena;
+			u64 run;
+		};
+		u64 values[3];
+	};
+};
+
+struct perf_counts {
+	s8			  scaled;
+	struct perf_counts_values aggr;
+	struct xyarray		  *values;
+};
+
+
+static inline struct perf_counts_values*
+perf_counts(struct perf_counts *counts, int cpu, int thread)
+{
+	return xyarray__entry(counts->values, cpu, thread);
+}
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads);
+void perf_counts__delete(struct perf_counts *counts);
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel);
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
+void perf_evsel__free_counts(struct perf_evsel *evsel);
+
+#endif /* __PERF_COUNTS_H */
tools/perf/util/evsel.h +1 -1
index 09a3022fa2c6..0ca0002db99a 100644
@@ -9,7 +9,7 @@
 #include "xyarray.h"
 #include "symbol.h"
 #include "cpumap.h"
-#include "stat.h"
+#include "counts.h"
 
 struct perf_evsel;
 
tools/perf/util/python-ext-sources +1 -1
index 0766d98c5da5..51be28b1bca2 100644
@@ -16,7 +16,7 @@ util/util.c
 util/xyarray.c
 util/cgroup.c
 util/rblist.c
-util/stat.c
+util/counts.c
 util/strlist.c
 util/trace-event.c
 ../lib/rbtree.c
tools/perf/util/stat.c +0 -49

55 lines not on this page · raw message

tools/perf/util/stat.h +0 -30
index 0b897b083682..62448c8175d3 100644
@@ -33,23 +33,6 @@ enum aggr_mode {
 	AGGR_THREAD,
 };
 
-struct perf_counts_values {
-	union {
-		struct {
-			u64 val;
-			u64 ena;
-			u64 run;
-		};
-		u64 values[3];
-	};
-};
-
-struct perf_counts {
-	s8			  scaled;
-	struct perf_counts_values aggr;
-	struct xyarray		  *values;
-};
-
 struct perf_stat_config {
 	enum aggr_mode	aggr_mode;
 	bool		scale;
@@ -57,12 +40,6 @@ struct perf_stat_config {
 	unsigned int	interval;
 };
 
-static inline struct perf_counts_values*
-perf_counts(struct perf_counts *counts, int cpu, int thread)
-{
-	return xyarray__entry(counts->values, cpu, thread);
-}
-
 void update_stats(struct stats *stats, u64 val);
 double avg_stats(struct stats *stats);
 double stddev_stats(struct stats *stats);
@@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
 void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
 				   double avg, int cpu, enum aggr_mode aggr);
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads);
-void perf_counts__delete(struct perf_counts *counts);
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel);
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
-void perf_evsel__free_counts(struct perf_evsel *evsel);
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
 int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
 void perf_evsel__free_stat_priv(struct perf_evsel *evsel);
#17 lore raw

Re: [PATCH] perf tools: Move perf_counts struct and functions into separate object

Em Fri, Aug 07, 2015 at 12:51:03PM +0200, Jiri Olsa escreveu:

Quoted 11 lines · On Thu, Aug 06, 2015 at 03:44:52PM -0400, [email protected] wrote:
Quoted 3 lines

From: Kan Liang <[email protected]>

Move callchain option parse related code to util.c

little nore about the reason would be nice ;-)

looks ok, but the python test is still failing, the reason is the perf_counts struct objects and functions bindings to util/stat.c object

Quoted 5 lines

Arnaldo,
I separated the 'struct perf_counts' into util/counts.c, because I recall you did not want them in evsel.c, please check attached patch.. it makes the python test pass again (on top of Kan's change).

Ok, will apply, try Kan's patch, add the extra comments about why the move from callchain to util.c is needed, test the whole shebang, ship to Ingo...

- Arnaldo

Quoted 279 lines

thanks,
jirka

---
Moving perf_counts struct and functions into separate object, so we could remove stat.c object dependency from python build.

It makes the python code to be built properly, because now it fails to load due to missing stat-shadow.c object dependency.

---

tools/perf/builtin-stat.c +1 -0
index a054ddc0b2a0..7aa039bd379a 100644
@@ -58,6 +58,7 @@
 #include "util/cpumap.h"
 #include "util/thread.h"
 #include "util/thread_map.h"
+#include "util/counts.h"
 
 #include <stdlib.h>
 #include <sys/prctl.h>
tools/perf/util/Build +1 -0
index a1e5168dc1fb..4ca481cd38e6 100644
@@ -67,6 +67,7 @@ libperf-y += target.o
 libperf-y += rblist.o
 libperf-y += intlist.o
 libperf-y += vdso.o
+libperf-y += counts.o
 libperf-y += stat.o
 libperf-y += stat-shadow.o
 libperf-y += record.o
tools/perf/util/counts.c new file +52 -0
new file mode 100644
index 000000000000..e3fde313deb2
@@ -0,0 +1,52 @@
+#include <stdlib.h>
+#include "evsel.h"
+#include "counts.h"
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads)
+{
+	struct perf_counts *counts = zalloc(sizeof(*counts));
+
+	if (counts) {
+		struct xyarray *values;
+
+		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
+		if (!values) {
+			free(counts);
+			return NULL;
+		}
+
+		counts->values = values;
+	}
+
+	return counts;
+}
+
+void perf_counts__delete(struct perf_counts *counts)
+{
+	if (counts) {
+		xyarray__delete(counts->values);
+		free(counts);
+	}
+}
+
+static void perf_counts__reset(struct perf_counts *counts)
+{
+	xyarray__reset(counts->values);
+}
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel)
+{
+	perf_counts__reset(evsel->counts);
+}
+
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
+{
+	evsel->counts = perf_counts__new(ncpus, nthreads);
+	return evsel->counts != NULL ? 0 : -ENOMEM;
+}
+
+void perf_evsel__free_counts(struct perf_evsel *evsel)
+{
+	perf_counts__delete(evsel->counts);
+	evsel->counts = NULL;
+}
tools/perf/util/counts.h new file +37 -0
new file mode 100644
index 000000000000..34d8baaf558a
@@ -0,0 +1,37 @@
+#ifndef __PERF_COUNTS_H
+#define __PERF_COUNTS_H
+
+#include "xyarray.h"
+
+struct perf_counts_values {
+	union {
+		struct {
+			u64 val;
+			u64 ena;
+			u64 run;
+		};
+		u64 values[3];
+	};
+};
+
+struct perf_counts {
+	s8			  scaled;
+	struct perf_counts_values aggr;
+	struct xyarray		  *values;
+};
+
+
+static inline struct perf_counts_values*
+perf_counts(struct perf_counts *counts, int cpu, int thread)
+{
+	return xyarray__entry(counts->values, cpu, thread);
+}
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads);
+void perf_counts__delete(struct perf_counts *counts);
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel);
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
+void perf_evsel__free_counts(struct perf_evsel *evsel);
+
+#endif /* __PERF_COUNTS_H */
tools/perf/util/evsel.h +1 -1
index 09a3022fa2c6..0ca0002db99a 100644
@@ -9,7 +9,7 @@
 #include "xyarray.h"
 #include "symbol.h"
 #include "cpumap.h"
-#include "stat.h"
+#include "counts.h"
 
 struct perf_evsel;
 
tools/perf/util/python-ext-sources +1 -1
index 0766d98c5da5..51be28b1bca2 100644
@@ -16,7 +16,7 @@ util/util.c
 util/xyarray.c
 util/cgroup.c
 util/rblist.c
-util/stat.c
+util/counts.c
 util/strlist.c
 util/trace-event.c
 ../lib/rbtree.c
tools/perf/util/stat.c +0 -49
index c5c709cdc3ce..415c359de465 100644
@@ -97,55 +97,6 @@ void perf_stat_evsel_id_init(struct perf_evsel *evsel)
 	}
 }
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads)
-{
-	struct perf_counts *counts = zalloc(sizeof(*counts));
-
-	if (counts) {
-		struct xyarray *values;
-
-		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
-		if (!values) {
-			free(counts);
-			return NULL;
-		}
-
-		counts->values = values;
-	}
-
-	return counts;
-}
-
-void perf_counts__delete(struct perf_counts *counts)
-{
-	if (counts) {
-		xyarray__delete(counts->values);
-		free(counts);
-	}
-}
-
-static void perf_counts__reset(struct perf_counts *counts)
-{
-	xyarray__reset(counts->values);
-}
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel)
-{
-	perf_counts__reset(evsel->counts);
-}
-
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
-{
-	evsel->counts = perf_counts__new(ncpus, nthreads);
-	return evsel->counts != NULL ? 0 : -ENOMEM;
-}
-
-void perf_evsel__free_counts(struct perf_evsel *evsel)
-{
-	perf_counts__delete(evsel->counts);
-	evsel->counts = NULL;
-}
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel)
 {
 	int i;
tools/perf/util/stat.h +0 -30
index 0b897b083682..62448c8175d3 100644
@@ -33,23 +33,6 @@ enum aggr_mode {
 	AGGR_THREAD,
 };
 
-struct perf_counts_values {
-	union {
-		struct {
-			u64 val;
-			u64 ena;
-			u64 run;
-		};
-		u64 values[3];
-	};
-};
-
-struct perf_counts {
-	s8			  scaled;
-	struct perf_counts_values aggr;
-	struct xyarray		  *values;
-};
-
 struct perf_stat_config {
 	enum aggr_mode	aggr_mode;
 	bool		scale;
@@ -57,12 +40,6 @@ struct perf_stat_config {
 	unsigned int	interval;
 };
 
-static inline struct perf_counts_values*
-perf_counts(struct perf_counts *counts, int cpu, int thread)
-{
-	return xyarray__entry(counts->values, cpu, thread);
-}
-
 void update_stats(struct stats *stats, u64 val);
 double avg_stats(struct stats *stats);
 double stddev_stats(struct stats *stats);
@@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
 void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
 				   double avg, int cpu, enum aggr_mode aggr);
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads);
-void perf_counts__delete(struct perf_counts *counts);
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel);
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
-void perf_evsel__free_counts(struct perf_evsel *evsel);
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
 int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
 void perf_evsel__free_stat_priv(struct perf_evsel *evsel);
#18 lore raw

[tip:perf/core] perf stat: Move perf_counts struct and functions into separate object

In reply to Jiri Olsa
perf stat: Move perf_counts struct and functions into separate object
Commit-ID:  d809560b36a7ed31fbaf3719fdf79ddcbd30950b
Gitweb:     http://git.kernel.org/tip/d809560b36a7ed31fbaf3719fdf79ddcbd30950b
Author:     Jiri Olsa <[email protected]>
AuthorDate: Fri, 7 Aug 2015 12:51:03 +0200
Committer:  Arnaldo Carvalho de Melo <[email protected]>
CommitDate: Sat, 8 Aug 2015 14:16:49 -0300

perf stat: Move perf_counts struct and functions into separate object

Moving 'struct perf_counts' and associated functions into separate object, so we could remove stat.c object dependency from python build.

It makes the python code to build properly, because it fails to load due to missing stat-shadow.c object dependency if some patches from Kan Liang are applied.

So apply this one, then Kan's.

---
 tools/perf/builtin-stat.c          |  1 +
 tools/perf/util/Build              |  1 +
 tools/perf/util/counts.c           | 52 ++++++++++++++++++++++++++++++++++++++
 tools/perf/util/counts.h           | 37 +++++++++++++++++++++++++++
 tools/perf/util/evsel.h            |  2 +-
 tools/perf/util/python-ext-sources |  2 +-
 tools/perf/util/stat.c             | 49 -----------------------------------
 tools/perf/util/stat.h             | 30 ----------------------
 8 files changed, 93 insertions(+), 81 deletions(-)
tools/perf/builtin-stat.c +1 -0
index a054ddc..7aa039b 100644
@@ -58,6 +58,7 @@
 #include "util/cpumap.h"
 #include "util/thread.h"
 #include "util/thread_map.h"
+#include "util/counts.h"
 
 #include <stdlib.h>
 #include <sys/prctl.h>
tools/perf/util/Build +1 -0
index 2ee81d7..1ce0adc 100644
@@ -68,6 +68,7 @@ libperf-y += target.o
 libperf-y += rblist.o
 libperf-y += intlist.o
 libperf-y += vdso.o
+libperf-y += counts.o
 libperf-y += stat.o
 libperf-y += stat-shadow.o
 libperf-y += record.o
tools/perf/util/counts.c new file +52 -0

52 lines not on this page · raw message

tools/perf/util/counts.h new file +37 -0
new file mode 100644
index 0000000..34d8baa
@@ -0,0 +1,37 @@
+#ifndef __PERF_COUNTS_H
+#define __PERF_COUNTS_H
+
+#include "xyarray.h"
+
+struct perf_counts_values {
+	union {
+		struct {
+			u64 val;
+			u64 ena;
+			u64 run;
+		};
+		u64 values[3];
+	};
+};
+
+struct perf_counts {
+	s8			  scaled;
+	struct perf_counts_values aggr;
+	struct xyarray		  *values;
+};
+
+
+static inline struct perf_counts_values*
+perf_counts(struct perf_counts *counts, int cpu, int thread)
+{
+	return xyarray__entry(counts->values, cpu, thread);
+}
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads);
+void perf_counts__delete(struct perf_counts *counts);
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel);
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
+void perf_evsel__free_counts(struct perf_evsel *evsel);
+
+#endif /* __PERF_COUNTS_H */
tools/perf/util/evsel.h +1 -1
index 6a12908..b948f69 100644
@@ -9,7 +9,7 @@
 #include "xyarray.h"
 #include "symbol.h"
 #include "cpumap.h"
-#include "stat.h"
+#include "counts.h"
 
 struct perf_evsel;
 
tools/perf/util/python-ext-sources +1 -1
index 0766d98..51be28b 100644
@@ -16,7 +16,7 @@ util/util.c
 util/xyarray.c
 util/cgroup.c
 util/rblist.c
-util/stat.c
+util/counts.c
 util/strlist.c
 util/trace-event.c
 ../lib/rbtree.c
tools/perf/util/stat.c +0 -49

55 lines not on this page · raw message

tools/perf/util/stat.h +0 -30
index 0b897b0..62448c8 100644
@@ -33,23 +33,6 @@ enum aggr_mode {
 	AGGR_THREAD,
 };
 
-struct perf_counts_values {
-	union {
-		struct {
-			u64 val;
-			u64 ena;
-			u64 run;
-		};
-		u64 values[3];
-	};
-};
-
-struct perf_counts {
-	s8			  scaled;
-	struct perf_counts_values aggr;
-	struct xyarray		  *values;
-};
-
 struct perf_stat_config {
 	enum aggr_mode	aggr_mode;
 	bool		scale;
@@ -57,12 +40,6 @@ struct perf_stat_config {
 	unsigned int	interval;
 };
 
-static inline struct perf_counts_values*
-perf_counts(struct perf_counts *counts, int cpu, int thread)
-{
-	return xyarray__entry(counts->values, cpu, thread);
-}
-
 void update_stats(struct stats *stats, u64 val);
 double avg_stats(struct stats *stats);
 double stddev_stats(struct stats *stats);
@@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
 void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
 				   double avg, int cpu, enum aggr_mode aggr);
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads);
-void perf_counts__delete(struct perf_counts *counts);
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel);
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
-void perf_evsel__free_counts(struct perf_evsel *evsel);
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
 int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
 void perf_evsel__free_stat_priv(struct perf_evsel *evsel);
#19 lore raw

[tip:perf/core] perf callchain: Move option parsing code to util.c

In reply to [email protected]
perf callchain: Move option parsing code to util.c
Commit-ID:  076a30c411ba2b91a18d44a5a01977035cdd7716
Gitweb:     http://git.kernel.org/tip/076a30c411ba2b91a18d44a5a01977035cdd7716
Author:     Kan Liang <[email protected]>
AuthorDate: Thu, 6 Aug 2015 15:44:52 -0400
Committer:  Arnaldo Carvalho de Melo <[email protected]>
CommitDate: Sat, 8 Aug 2015 14:16:49 -0300

perf callchain: Move option parsing code to util.c

Move callchain option parse related code to util.c, to avoid dragging more object files into the python binding.

---
 tools/perf/util/callchain.c | 89 +-------------------------------------------
 tools/perf/util/callchain.h |  1 +
 tools/perf/util/util.c      | 90 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/util.h      |  2 +
 4 files changed, 94 insertions(+), 88 deletions(-)
tools/perf/util/callchain.c +1 -88
index 931cca8..773fe13 100644
@@ -25,96 +25,9 @@
 
 __thread struct callchain_cursor callchain_cursor;
 
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-static int get_stack_size(const char *str, unsigned long *_size)
-{
-	char *endptr;
-	unsigned long size;
-	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
-
-	size = strtoul(str, &endptr, 0);
-
-	do {
-		if (*endptr)
-			break;
-
-		size = round_up(size, sizeof(u64));
-		if (!size || size > max_size)
-			break;
-
-		*_size = size;
-		return 0;
-
-	} while (0);
-
-	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
-	       max_size, str);
-	return -1;
-}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param)
 {
-	char *tok, *name, *saveptr = NULL;
-	char *buf;
-	int ret = -1;
-
-	/* We need buffer that we know we can write to. */
-	buf = malloc(strlen(arg) + 1);
-	if (!buf)
-		return -ENOMEM;
-
-	strcpy(buf, arg);
-
-	tok = strtok_r((char *)buf, ",", &saveptr);
-	name = tok ? : (char *)buf;
-
-	do {
-		/* Framepointer style */
-		if (!strncmp(name, "fp", sizeof("fp"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_FP;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-				       "needed for --call-graph fp\n");
-			break;
-
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-		/* Dwarf style */
-		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
-			const unsigned long default_stack_dump_size = 8192;
-
-			ret = 0;
-			param->record_mode = CALLCHAIN_DWARF;
-			param->dump_size = default_stack_dump_size;
-
-			tok = strtok_r(NULL, ",", &saveptr);
-			if (tok) {
-				unsigned long size = 0;
-
-				ret = get_stack_size(tok, &size);
-				param->dump_size = size;
-			}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_LBR;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-					"needed for --call-graph lbr\n");
-			break;
-		} else {
-			pr_err("callchain: Unknown --call-graph option "
-			       "value: %s\n", arg);
-			break;
-		}
-
-	} while (0);
-
-	free(buf);
-	return ret;
+	return parse_callchain_record(arg, param);
 }
 
 static int parse_callchain_mode(const char *value)
tools/perf/util/callchain.h +1 -0
index 68a32c2..acee2b3 100644
@@ -177,6 +177,7 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *
 			bool hide_unresolved);
 
 extern const char record_callchain_help[];
+extern int parse_callchain_record(const char *arg, struct callchain_param *param);
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param);
 int parse_callchain_report_opt(const char *arg);
 int perf_callchain_config(const char *var, const char *value);
tools/perf/util/util.c +90 -0

96 lines not on this page · raw message

tools/perf/util/util.h +2 -0
index 20d625a..8148703 100644
@@ -351,4 +351,6 @@ static inline char *asprintf_expr_not_in_ints(const char *var, size_t nints, int
 	return asprintf_expr_inout_ints(var, false, nints, ints);
 }
 
+int get_stack_size(const char *str, unsigned long *_size);
+
 #endif /* GIT_COMPAT_UTIL_H */