perf/sdt: Directly record cached SDT events

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

tools/perf/builtin-probe.c +2 -2

index 4a86aea..560cce3 100644
@@ -322,7 +322,7 @@ static int perf_add_probe_events(struct perf_probe_event *pevs, int npevs)
 	if (ret < 0)
 		return ret;
 
-	ret = convert_perf_probe_events(pevs, npevs);
+	ret = convert_perf_probe_events(pevs, npevs, false);
 	if (ret < 0)
 		goto out_cleanup;
 

@@ -387,7 +387,7 @@ static int del_perf_probe_caches(struct strfilter *filter)
 	return 0;
 }
 
-static int perf_del_probe_events(struct strfilter *filter)
+static int perf_del_probe_events(struct strfilter *filter, bool show)
 {
 	int ret, ret2, ufd = -1, kfd = -1;
 	char *str = strfilter__string(filter);

tools/perf/builtin-probe.c +23 -1

 		ret = probe_file__del_strlist(kfd, klist);
 		if (ret < 0)

@@ -427,7 +428,8 @@ static int perf_del_probe_events(struct strfilter *filter)
 	ret2 = probe_file__get_events(ufd, filter, ulist);
 	if (ret2 == 0) {
 		strlist__for_each(ent, ulist)
-			pr_info("Removed event: %s\n", ent->s);
+			if (show)
+				pr_info("Removed event: %s\n", ent->s);
 
 		ret2 = probe_file__del_strlist(ufd, ulist);
 		if (ret2 < 0)

@@ -452,6 +454,38 @@ out:
 	return ret;
 }
 
+/*
+ * Record session for SDT events has ended. Delete the SDT events
+ * from uprobe_events file that were created initially.
+ */
+void remove_sdt_event_list(struct list_head *sdt_events)
+{
+	struct sdt_event_list *event;
+	struct strfilter *filter = NULL;
+	const char *err = NULL;
+	int ret = 0;
+
+	if (list_empty(sdt_events))
+		return;
+
+	list_for_each_entry(event, sdt_events, list) {
+		if (!filter) {
+			filter = strfilter__new(event->event_name, &err);
+			if (!filter)
+				goto free_list;
+		} else {
+			ret = strfilter__or(filter, event->event_name, &err);

tools/perf/util/parse-events.c +19 -1

index 4033dce..cc017a5 100644
@@ -1687,7 +1687,41 @@ int parse_events_option(const struct option *opt, const char *str,
 {
 	struct perf_evlist *evlist = *(struct perf_evlist **)opt->value;
 	struct parse_events_error err = { .idx = 0, };
-	int ret = parse_events(evlist, str, &err);
+	int ret = 0;
+
+	if (*str == '%' && is_cmd_record()) {
+		char *ptr = NULL;
+		struct list_head *sdt_list;
+
+		ptr = zalloc(strlen(str));
+		if (ptr == NULL)
+			return -ENOMEM;
+		strcpy(ptr, str);
+
+		sdt_list = zalloc(sizeof(*sdt_list));
+		if (!sdt_list) {
+			free(ptr);
+			pr_err("Error in sdt_list memory allocation\n");
+			return -ENOMEM;
+		}
+		INIT_LIST_HEAD(sdt_list);
+

tools/perf/util/parse-events.c +6 -0

+	if (!ret) {
+		ret = parse_events(evlist, str + 1, &err);
+		if (ret > 0)
+			ret = 0;
+	} else
+		ret = parse_events(evlist, str, &err);
 
 	if (ret)
 		parse_events_print_error(&err, str);

tools/perf/util/parse-events.h +2 -0

index c08daa9..98503c6 100644
@@ -189,4 +189,6 @@ int is_valid_tracepoint(const char *event_string);
 int valid_event_mount(const char *eventfs);
 char *parse_events_formats_error_string(char *additional_terms);
 
+void sdt_event_list__add(struct list_head *sdt_event_list);
+bool is_cmd_record(void);
 #endif /* __PERF_PARSE_EVENTS_H */

tools/perf/util/probe-event.c +4 -2

index bb9fc34..34f6bab 100644
@@ -1144,7 +1144,7 @@ err:
 	return err;
 }
 
-static int parse_perf_probe_event_name(char **arg, struct perf_probe_event *pev)
+int parse_perf_probe_event_name(char **arg, struct perf_probe_event *pev)
 {
 	char *ptr;
 

@@ -3008,7 +3008,8 @@ out:
 }
 
 static int convert_to_probe_trace_events(struct perf_probe_event *pev,
-					 struct probe_trace_event **tevs)
+					 struct probe_trace_event **tevs,
+					 bool rec_enabled)
 {
 	int ret;
 

@@ -3033,6 +3034,17 @@ static int convert_to_probe_trace_events(struct perf_probe_event *pev,
 	ret = find_probe_trace_events_from_cache(pev, tevs);
 	if (ret > 0)
 		return ret;	/* Found in probe cache */
+	else if (rec_enabled) {

tools/perf/util/probe-event.c +39 -0

+/*
+ * Find the SDT event from the cache and if found add it/them
+ * to the uprobe_events file
+ */
+int add_sdt_event(char *event, struct list_head *sdt_events)
+{
+	struct perf_probe_event *pev;
+	int ret, i;
+	char *str = event + 1;
+	struct sdt_event_list *tmp;
+
+	pev = zalloc(sizeof(*pev));
+	if (!pev)
+		return -ENOMEM;
+
+	pev->sdt = true;
+	pev->uprobes = true;
+
+	/*
+	 * Parse str to find the group name and event name of
+	 * the sdt event.
+	 */
+	ret = parse_perf_probe_event_name(&str, pev);
+	if (ret) {
+		pr_err("Error in parsing sdt event %s\n", str);
+		free(pev);
+		return ret;
+	}
+
+	probe_conf.max_probes = MAX_PROBES;
+	probe_conf.force_add = 1;
+
+	/*
+	 * Find the sdt event from the cache, only cached SDT
+	 * events can be directly recorded.
+	 */
+	ret = convert_perf_probe_events(pev, 1, true);
+	if (ret < 0) {
+		goto free_pev;

tools/perf/util/probe-event.c +13 -0

+			if (!tmp->event_name) {
+				free_sdt_list(sdt_events);
+				ret = -ENOMEM;
+				goto free_pev;
+			}
+			list_add(&tmp->list, sdt_events);
+		}
+	}
+
+free_pev:
+	cleanup_perf_probe_events(pev, 1);
+	return 0;
+}

tools/perf/util/probe-file.h +0 -0

index a02bbbd..f120614 100644

Hi Hemant,

Quoted 9 lines · On Fri, 29 Apr 2016 19:10:41 +0530 Hemant Kumar <[email protected]>…

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Quoted 3 lines

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

Thanks! However, before looking over each part of this patch, I think this is not enough for supporting SDT for perf record.

If there are several SDTs which have same eventname but differnt addresses (e.g. libc:memory_memalign_retry), how are those handled? Currently, to support this, we'll need to enable those events in different names, or just pick one of them. It could confuse users in each case.

To solve this issue, we need to introduce multiple SDTs on single ftrace event. Please read my comment on v3 patch (https://lkml.org/lkml/2015/8/15/52)

So, at this point, I've decided to introduce only perf-probe side. If user want to record SDT, they can use perf-probe to add SDT events and see what events are generated by SDT, so that they can specify those events via perf-record.
e.g.

# perf probe -a %sdt_libc:*
...
   sdt_libc:lll_futex_wake_1 (on %* in /usr/lib64/libc-2.20.so)
   sdt_libc:lll_lock_wait_private (on %* in /usr/lib64/libc-2.20.so)

You can now use it in all perf tools, such as:

perf record -e sdt_libc:lll_lock_wait_private -aR sleep 1

# perf record -e sdt_libc:* -a
^C[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 1.461 MB perf.data (6195 samples) ]

# perf script
  plugin_audio_th 11119 [000] 16059.864654:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881345:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881350:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881411:   sdt_libc:setjmp: (7f37bf55b6c1)
...

BTW, see below comments on the code for implementation issues.

Hi Masami,

Quoted 14 lines · Masami Hiramatsu

Hi Hemant,

Quoted 9 lines · On Fri, 29 Apr 2016 19:10:41 +0530 Hemant Kumar <[email protected]>…

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Quoted 5 lines

Quoted 3 lines

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

Thanks! However, before looking over each part of this patch, I think this is not enough for supporting SDT for perf record.

Hmm.

Quoted 5 lines

If there are several SDTs which have same eventname but differnt addresses (e.g. libc:memory_memalign_retry), how are those handled? Currently, to support this, we'll need to enable those events in different names, or just pick one of them. It could confuse users in each case.

Right. But now, its the same case with a binary having multiple symbols with same names, isn't it?

# nm ./multi | grep foo
0000000000400530 t foo
0000000000400560 t foo

# perf probe -x ./multi foo
Added new events:
   probe_multi:foo      (on foo in /home/hemant/work/linux/tools/perf/multi)
   probe_multi:foo_1    (on foo in /home/hemant/work/linux/tools/perf/multi)

You can now use it in all perf tools, such as:

perf record -e probe_multi:foo_1 -aR sleep 1

My point being, the user can still know, if its shown that there are two or more probes being placed and the o/p of perf report/script shows that the probes are placed at two or more different addresses.

Quoted 2 lines

To solve this issue, we need to introduce multiple SDTs on single ftrace event. Please read my comment on v3 patch (https://lkml.org/lkml/2015/8/15/52)

Ok. But, I think, for initial direct recording support, we can go with this IMHO.

Quoted 19 lines

So, at this point, I've decided to introduce only perf-probe side. If user want to record SDT, they can use perf-probe to add SDT events and see what events are generated by SDT, so that they can specify those events via perf-record.
e.g.

# perf probe -a %sdt_libc:*
...
   sdt_libc:lll_futex_wake_1 (on %* in /usr/lib64/libc-2.20.so)
   sdt_libc:lll_lock_wait_private (on %* in /usr/lib64/libc-2.20.so)

You can now use it in all perf tools, such as:

perf record -e sdt_libc:lll_lock_wait_private -aR sleep 1

# perf record -e sdt_libc:* -a
^C[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 1.461 MB perf.data (6195 samples) ]

Quoted 8 lines

# perf script
  plugin_audio_th 11119 [000] 16059.864654:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881345:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881350:   sdt_libc:setjmp: (7f37bf55b6c1)
           chrome  4650 [000] 16059.881411:   sdt_libc:setjmp: (7f37bf55b6c1)
...

BTW, see below comments on the code for implementation issues.

Will fix the issues and send a v2.

Thanks a lot for the review.

Signature · 2 lines

Thanks,
Hemant Kumar

Quoted 29 lines · On Tue, 03 May 2016 05:06:24 +0530 Hemant Kumar <[email protected]>…

Hi Masami,

Quoted 14 lines · Masami Hiramatsu

Hi Hemant,

Quoted 9 lines · On Fri, 29 Apr 2016 19:10:41 +0530 Hemant Kumar <[email protected]>…

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Quoted 5 lines

Quoted 3 lines

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

Thanks! However, before looking over each part of this patch, I think this is not enough for supporting SDT for perf record.

Hmm.

Quoted 5 lines

If there are several SDTs which have same eventname but differnt addresses (e.g. libc:memory_memalign_retry), how are those handled? Currently, to support this, we'll need to enable those events in different names, or just pick one of them. It could confuse users in each case.

Quoted 2 lines

Right. But now, its the same case with a binary having multiple symbols with same names, isn't it?

Yes, but for the symbols or lines etc., user can not directly specify it via perf record. And as you showed below, perf-probe expresses there are 2 events on the probe point. So user is forced to aware of it.

Quoted 9 lines

# nm ./multi | grep foo
0000000000400530 t foo
0000000000400560 t foo

# perf probe -x ./multi foo
Added new events:
    probe_multi:foo      (on foo in /home/hemant/work/linux/tools/perf/multi)
    probe_multi:foo_1    (on foo in /home/hemant/work/linux/tools/perf/multi)

Quoted 8 lines

You can now use it in all perf tools, such as:

perf record -e probe_multi:foo_1 -aR sleep 1

My point being, the user can still know, if its shown that there are two or more probes being placed and the o/p of perf report/script shows that the probes are placed at two or more different addresses.

Not only the different address, but also they will see the different event names. That may be no good for making a script on it.

My point is, if the user only uses "perf record -e sdt_something:sdtevent", they will think that there is one event recorded. it can easily misleading them.

Quoted 4 lines

Quoted 2 lines

To solve this issue, we need to introduce multiple SDTs on single ftrace event. Please read my comment on v3 patch (https://lkml.org/lkml/2015/8/15/52)

Ok. But, I think, for initial direct recording support, we can go with this IMHO.

So, at least this should be noticed to users carefully. (e.g. warn if there are more than two SDTs defined)

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

For example :

 # perf list sdt       // List the SDT events
...
  sdt_mysql:update__row__done                        [SDT event]
  sdt_mysql:update__row__start                       [SDT event]
  sdt_mysql:update__start                            [SDT event]
  sdt_python:function__entry                         [SDT event]
  sdt_python:function__return                        [SDT event]
  sdt_test:marker1                                   [SDT event]
  sdt_test:marker2                                   [SDT event]
...

 # perf record -e %sdt_test:marker1 -e %sdt_test:marker2 -a

^C[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 2.087 MB perf.data (22 samples) ]

 # perf script
        test_sdt 29230 [002] 405550.548017: sdt_test:marker1: (400534)
        test_sdt 29230 [002] 405550.548064: sdt_test:marker2: (40053f)
        test_sdt 29231 [002] 405550.962806: sdt_test:marker1: (400534)
        test_sdt 29231 [002] 405550.962841: sdt_test:marker2: (40053f)
        test_sdt 29232 [001] 405551.379327: sdt_test:marker1: (400534)
 ...

After invoking "perf record", behind the scenes, it checks whether the event specified is an SDT event using the flag '%'. After that, it does a lookup of the probe cache to find out the SDT event. If its not present, it throws an error. Otherwise, it goes on and writes the event into the uprobe_events file and sets up the probe event, trace events, etc and starts recording. It also maintains a list of the event names that were written to uprobe_events file. After finishing the record session, it removes the events from the uprobe_events file using the maintained name list.

Quoted 26 lines · Hemant Kumar

This patch adds support for directly recording SDT events which are present in the probe cache. This patch is based on current SDT enablement patchset (v5) by Masami :
https://lkml.org/lkml/2016/4/27/828
and it implements two points in the TODO list mentioned in the cover note :
"- (perf record) Support SDT event recording directly" "- (perf record) Try to unregister SDT events after record."

Without this patch, we could probe into SDT events using "perf probe" and "perf record". With this patch, we can probe the SDT events directly using "perf record".

For example :

 # perf list sdt       // List the SDT events
...
  sdt_mysql:update__row__done                        [SDT event]
  sdt_mysql:update__row__start                       [SDT event]
  sdt_mysql:update__start                            [SDT event]
  sdt_python:function__entry                         [SDT event]
  sdt_python:function__return                        [SDT event]
  sdt_test:marker1                                   [SDT event]
  sdt_test:marker2                                   [SDT event]
...

Quoted 1 line

 # perf record -e %sdt_test:marker1 -e %sdt_test:marker2 -a

Why do we need the '%'? Can't the "sdt_" prefix be sufficient? ie:

# perf record -e sdt_test:marker1 -e sdt_test:marker2 -a

I find it a bit weird to define it using %sdt_, but then use it using sdt_. I'd also be inclined to use it for probe creation, ie:

# perf probe -x /lib/libc-2.17.so  sdt_libc:lll_lock_wait_private

That way, the user only learns one way to specify the probe, with the sdt_ prefix. It's fine if % existed too, but optional.

Quoted 11 lines

^C[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 2.087 MB perf.data (22 samples) ]

 # perf script
        test_sdt 29230 [002] 405550.548017: sdt_test:marker1: (400534)
        test_sdt 29230 [002] 405550.548064: sdt_test:marker2: (40053f)
        test_sdt 29231 [002] 405550.962806: sdt_test:marker1: (400534)
        test_sdt 29231 [002] 405550.962841: sdt_test:marker2: (40053f)
        test_sdt 29232 [001] 405551.379327: sdt_test:marker1: (400534)
 ...

Quoted 9 lines

After invoking "perf record", behind the scenes, it checks whether the event specified is an SDT event using the flag '%'. After that, it does a lookup of the probe cache to find out the SDT event. If its not present, it throws an error. Otherwise, it goes on and writes the event into the uprobe_events file and sets up the probe event, trace events, etc and starts recording. It also maintains a list of the event names that were written to uprobe_events file. After finishing the record session, it removes the events from the uprobe_events file using the maintained name list.

Does this support semaphore SDT probes (is-enabled)? Those need the semaphore incremented when enabled, then decremented when disabled.