Provide additional sample information to Python scripts

Good morning,

We have developed a patch for the perf Python scripting interface to provide additional information about the pid, tid, and cpu of generic events as well as information about the call-stack and resolved symbol names. This provides scripts with a greater level of detail. The mentioned information is already available to the scripting engine and just has to be handed down. This is done by the attached patch. The patch is based on Linux-3.13.3.

Please let me know if you have any questions on this.

Thanks
Joseph

Signature · 23 lines

Dipl. Inf. Joseph Schuchart
Computer Scientist

Technische Universität Dresden
Center for Information Services and High Performance Computing (ZIH)
01062 Dresden, Germany

Phone: +49 351 463-36494
Fax: +49 351 463-3773
E-Mail: [email protected]

Perf: Provide sample information and call-chain to Python script

Provide additional sample information on generic events to Python 
scripts, including pid, tid, and cpu for which the event was recorded.
Additionally, provide the call-stack recorded at each event with 
resolved symbols. At the moment, the pointer to the sample struct
is passed to scripts, which seems to be of little use. The patch
puts this information in dictionaries for easy access by Python
scripts. 

Signed-off-by: Joseph Schuchart <[email protected]>
Acked-by: Thomas Ilsche <[email protected]>

(quoted diff) +76 -3

@@ -359,7 +359,7 @@ static void python_process_general_event
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict;
+	PyObject *handler, *retval, *t, *dict, *dict_sample;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -375,6 +375,10 @@ static void python_process_general_event
 	if (!dict)
 		Py_FatalError("couldn't create Python dictionary");
 
+	dict_sample = PyDict_New();
+	if (!dict_sample)
+		Py_FatalError("couldn't create Python dictionary");
+
 	snprintf(handler_name, sizeof(handler_name), "%s", "process_event");
 
 	handler = PyDict_GetItemString(main_dict, handler_name);

@@ -384,8 +388,76 @@ static void python_process_general_event
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	/* PID/TIDs are limited to 2^29, so we can safely use PyInt */
+	pydict_set_item_string_decref(dict_sample, "pid", PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid", PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu", PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "time", PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);
+
+	/* ip unwinding */
+
+	if (symbol_conf.use_callchain && sample->callchain) {
+		PyObject *pylist;
+
+		if (machine__resolve_callchain(machine, evsel, al->thread,
+						   sample, NULL, NULL, PERF_MAX_STACK_DEPTH) != 0) {
+			pr_err("Failed to resolve callchain. Skipping\n");
+			goto exit;
+		}
+		callchain_cursor_commit(&callchain_cursor);
+
+		pylist = PyList_New(0);
+		if (!pylist)
+			Py_FatalError("couldn't create Python list");
+
+		while (1) {
+			PyObject *pyelem;
+			struct callchain_cursor_node *node;
+			node = callchain_cursor_current(&callchain_cursor);
+			if (!node)
+				break;
+
+			pyelem = PyDict_New();
+			if (!pyelem)
+				Py_FatalError("couldn't create Python dictionary");
+
+
+			pydict_set_item_string_decref(pyelem, "ip", PyInt_FromLong(node->ip));
+
+			if (node->sym) {
+				PyObject *pysym  = PyDict_New();
+				if (!pysym)
+					Py_FatalError("couldn't create Python dictionary");
+				pydict_set_item_string_decref(pysym, "start",   PyInt_FromLong(node->sym->start));
+				pydict_set_item_string_decref(pysym, "end",     PyInt_FromLong(node->sym->end));
+				pydict_set_item_string_decref(pysym, "binding", PyInt_FromLong(node->sym->binding));
+				pydict_set_item_string_decref(pysym, "name",    PyString_FromStringAndSize(node->sym->name, node->sym->namelen));
+				pydict_set_item_string_decref(pyelem, "sym", pysym);
+				Py_DECREF(pysym);
+			}
+
+			if (node->map) {
+				struct map *map = node->map;
+				const char *dsoname = "[unknown]";
+				if (map && map->dso && (map->dso->name || map->dso->long_name)) {
+					if (symbol_conf.show_kernel_path && map->dso->long_name)
+						dsoname = map->dso->long_name;
+					else if (map->dso->name)
+						dsoname = map->dso->name;
+				}
+				pydict_set_item_string_decref(pyelem, "dso", PyString_FromString(dsoname));
+			}
+
+			callchain_cursor_advance(&callchain_cursor);
+			PyList_Append(pylist, pyelem);
+			Py_DECREF(pyelem);
+		}
+		PyDict_SetItemString(dict, "callstack", pylist);
+		Py_DECREF(pylist);
+	}
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",

@@ -407,6 +479,7 @@ static void python_process_general_event
 	if (retval == NULL)
 		handler_call_die(handler_name);
 exit:
+	Py_DECREF(dict_sample);
 	Py_DECREF(dict);
 	Py_DECREF(t);
 }

Em Tue, Feb 18, 2014 at 09:43:24AM +0100, Joseph Schuchart escreveu:

Quoted 10 lines

Good morning,

We have developed a patch for the perf Python scripting interface to provide additional information about the pid, tid, and cpu of generic events as well as information about the call-stack and resolved symbol names. This provides scripts with a greater level of detail. The mentioned information is already available to the scripting engine and just has to be handed down. This is done by the attached patch. The patch is based on Linux-3.13.3.

Quoted 1 line

Please let me know if you have any questions on this.

Can you please resend, against the perf/core branch in
git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git, and as an attachement or making sure that the patch is not mangled?

- Arnaldo

Quoted 134 lines

Thanks
Joseph

Signature · 23 lines

Dipl. Inf. Joseph Schuchart
Computer Scientist

Technische Universität Dresden
Center for Information Services and High Performance Computing (ZIH)
01062 Dresden, Germany

Phone: +49 351 463-36494
Fax: +49 351 463-3773
E-Mail: [email protected]

Perf: Provide sample information and call-chain to Python script

Provide additional sample information on generic events to Python 
scripts, including pid, tid, and cpu for which the event was recorded.
Additionally, provide the call-stack recorded at each event with 
resolved symbols. At the moment, the pointer to the sample struct
is passed to scripts, which seems to be of little use. The patch
puts this information in dictionaries for easy access by Python
scripts. 

Signed-off-by: Joseph Schuchart <[email protected]>
Acked-by: Thomas Ilsche <[email protected]>

(quoted diff) +76 -3

@@ -359,7 +359,7 @@ static void python_process_general_event
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict;
+	PyObject *handler, *retval, *t, *dict, *dict_sample;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -375,6 +375,10 @@ static void python_process_general_event
 	if (!dict)
 		Py_FatalError("couldn't create Python dictionary");
 
+	dict_sample = PyDict_New();
+	if (!dict_sample)
+		Py_FatalError("couldn't create Python dictionary");
+
 	snprintf(handler_name, sizeof(handler_name), "%s", "process_event");
 
 	handler = PyDict_GetItemString(main_dict, handler_name);

@@ -384,8 +388,76 @@ static void python_process_general_event
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	/* PID/TIDs are limited to 2^29, so we can safely use PyInt */
+	pydict_set_item_string_decref(dict_sample, "pid", PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid", PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu", PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "time", PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);
+
+	/* ip unwinding */
+
+	if (symbol_conf.use_callchain && sample->callchain) {
+		PyObject *pylist;
+
+		if (machine__resolve_callchain(machine, evsel, al->thread,
+						   sample, NULL, NULL, PERF_MAX_STACK_DEPTH) != 0) {
+			pr_err("Failed to resolve callchain. Skipping\n");
+			goto exit;
+		}
+		callchain_cursor_commit(&callchain_cursor);
+
+		pylist = PyList_New(0);
+		if (!pylist)
+			Py_FatalError("couldn't create Python list");
+
+		while (1) {
+			PyObject *pyelem;
+			struct callchain_cursor_node *node;
+			node = callchain_cursor_current(&callchain_cursor);
+			if (!node)
+				break;
+
+			pyelem = PyDict_New();
+			if (!pyelem)
+				Py_FatalError("couldn't create Python dictionary");
+
+
+			pydict_set_item_string_decref(pyelem, "ip", PyInt_FromLong(node->ip));
+
+			if (node->sym) {
+				PyObject *pysym  = PyDict_New();
+				if (!pysym)
+					Py_FatalError("couldn't create Python dictionary");
+				pydict_set_item_string_decref(pysym, "start",   PyInt_FromLong(node->sym->start));
+				pydict_set_item_string_decref(pysym, "end",     PyInt_FromLong(node->sym->end));
+				pydict_set_item_string_decref(pysym, "binding", PyInt_FromLong(node->sym->binding));
+				pydict_set_item_string_decref(pysym, "name",    PyString_FromStringAndSize(node->sym->name, node->sym->namelen));
+				pydict_set_item_string_decref(pyelem, "sym", pysym);
+				Py_DECREF(pysym);
+			}
+
+			if (node->map) {
+				struct map *map = node->map;
+				const char *dsoname = "[unknown]";
+				if (map && map->dso && (map->dso->name || map->dso->long_name)) {
+					if (symbol_conf.show_kernel_path && map->dso->long_name)
+						dsoname = map->dso->long_name;
+					else if (map->dso->name)
+						dsoname = map->dso->name;
+				}
+				pydict_set_item_string_decref(pyelem, "dso", PyString_FromString(dsoname));
+			}
+
+			callchain_cursor_advance(&callchain_cursor);
+			PyList_Append(pylist, pyelem);
+			Py_DECREF(pyelem);
+		}
+		PyDict_SetItemString(dict, "callstack", pylist);
+		Py_DECREF(pylist);
+	}
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",

@@ -407,6 +479,7 @@ static void python_process_general_event
 	if (retval == NULL)
 		handler_call_die(handler_name);
 exit:
+	Py_DECREF(dict_sample);
 	Py_DECREF(dict);
 	Py_DECREF(t);
 }

Quoted 13 lines

Em Tue, Feb 18, 2014 at 09:43:24AM +0100, Joseph Schuchart escreveu:

Quoted 11 lines

Good morning,

We have developed a patch for the perf Python scripting interface to provide additional information about the pid, tid, and cpu of generic events as well as information about the call-stack and resolved symbol names. This provides scripts with a greater level of detail. The mentioned information is already available to the scripting engine and just has to be handed down. This is done by the attached patch. The patch is based on Linux-3.13.3.

Please let me know if you have any questions on this.

Quoted 3 lines

Can you please resend, against the perf/core branch in
git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git, and as an attachement or making sure that the patch is not mangled?

Joseph and I are working on this but due to vacation we expect to send you the patch with some improvements in the beginning of April. Please let us know in case there is some deadline.

Can you please resend, against the perf/core branch in
git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git, and as an attachement or making sure that the patch is not mangled?

static void python_process_tracepoint(struct perf_sample *sample,

(quoted diff) +2 -1

 				      struct perf_evsel *evsel,
 				      struct thread *thread,
 				      struct addr_location *al)
 {
-	PyObject *handler, *retval, *context, *t, *obj, *dict = NULL;
+	PyObject *handler, *retval, *context, *t, *obj, *callchain;
+	PyObject *dict = NULL;
 	static char handler_name[256];
 	struct format_field *field;
 	unsigned long long val;

(quoted diff) +1 -0

+
 	if (!handler)
 		PyTuple_SetItem(t, n++, dict);
 

(quoted diff) +2 -2

@@ -385,15 +476,30 @@ static void python_process_general_event(struct perf_sample *sample,
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	pydict_set_item_string_decref(dict_sample, "pid",

(quoted diff) +0 -0

 	}
 	if (al->sym) {
 		pydict_set_item_string_decref(dict, "symbol",

Perf: Fix possible memory leaks in Python interface

The function PyObject_CallObject() returns a new PyObject reference on which Py_DECREF has to be called to avoid memory leaks. This patch adds these calls where necessary.

Signed-off-by Joseph Schuchart <[email protected]>

tools/perf/util/scripting-engines/trace-event-python.c +0 -0

index cd9774d..ee17f64 100644

tools/perf/util/scripting-engines/trace-event-python.c +10 -2

 	}
 
 	Py_DECREF(t);

@@ -143,6 +145,8 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -333,6 +337,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {

@@ -340,6 +346,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			else
+				Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}

@@ -399,6 +407,8 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	else
+		Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);

@@ -444,8 +454,8 @@ static int run_start_sub(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_begin");
-
-	Py_DECREF(retval);
+	else
+		Py_DECREF(retval);
 	return err;
 error:
 	Py_XDECREF(main_dict);

Add missing calls to Py_DECREF.

The function PyObject_CallObject() returns a new PyObject reference on which Py_DECREF has to be called to avoid memory leaks. This patch adds these calls where necessary.

It also marks handler_call_die() as NORETURN to make this clear and to avoid unnecessary else statements. Note that the abort() is necessary since Py_FatalError() does not return but seems to be missing the __noretun__ attribute in Python versions <=2.7

tools/perf/util/scripting-engines/trace-event-python.c +7 -2

+	abort();
 }
 
 /*

@@ -97,6 +101,7 @@ static void define_value(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -143,6 +148,7 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -333,6 +339,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {

@@ -340,6 +347,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}

@@ -399,6 +407,7 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);

@@ -520,8 +529,7 @@ static int python_stop_script(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_end");
-	else
-		Py_DECREF(retval);
+	Py_DECREF(retval);
 out:
 	Py_XDECREF(main_dict);
 	Py_XDECREF(main_module);

Add callchain to generic and tracepoint events. Since this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

---
 .../python/Perf-Trace-Util/lib/Perf/Trace/Core.py  |   3 +-
 tools/perf/scripts/python/check-perf-trace.py      |   4 +-
 .../perf/scripts/python/failed-syscalls-by-pid.py  |   2 +-
 tools/perf/scripts/python/futex-contention.py      |   4 +-
 tools/perf/scripts/python/net_dropmonitor.py       |   2 +-
 tools/perf/scripts/python/netdev-times.py          |  26 ++---
 tools/perf/scripts/python/sched-migration.py       |  41 ++++----
 tools/perf/scripts/python/sctop.py                 |   2 +-
 tools/perf/scripts/python/syscall-counts-by-pid.py |   2 +-
 tools/perf/scripts/python/syscall-counts.py        |   2 +-
 .../util/scripting-engines/trace-event-python.c    | 106 ++++++++++++++++++++-
 11 files changed, 146 insertions(+), 48 deletions(-)

tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py +2 -1

index de7211e..38dfb72 100644
@@ -107,12 +107,13 @@ def taskState(state):
 
 class EventHeaders:
 	def __init__(self, common_cpu, common_secs, common_nsecs,
-		     common_pid, common_comm):
+		     common_pid, common_comm, common_callchain):
 		self.cpu = common_cpu
 		self.secs = common_secs
 		self.nsecs = common_nsecs
 		self.pid = common_pid
 		self.comm = common_comm
+		self.callchain = common_callchain
 
 	def ts(self):
 		return (self.secs * (10 ** 9)) + self.nsecs

tools/perf/scripts/python/check-perf-trace.py +2 -2

index 4647a76..334599c 100644
@@ -27,7 +27,7 @@ def trace_end():
 
 def irq__softirq_entry(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	vec):
+	common_callchain, vec):
 		print_header(event_name, common_cpu, common_secs, common_nsecs,
 			common_pid, common_comm)
 

@@ -38,7 +38,7 @@ def irq__softirq_entry(event_name, context, common_cpu,
 
 def kmem__kmalloc(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	call_site, ptr, bytes_req, bytes_alloc,
+	common_callchain, call_site, ptr, bytes_req, bytes_alloc,
 	gfp_flags):
 		print_header(event_name, common_cpu, common_secs, common_nsecs,
 			common_pid, common_comm)

tools/perf/scripts/python/failed-syscalls-by-pid.py +1 -1

index 85805fa..24c97a5 100644
@@ -39,7 +39,7 @@ def trace_end():
 
 def raw_syscalls__sys_exit(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, ret):
+	common_callchain, id, ret):
 	if (for_comm and common_comm != for_comm) or \
 	   (for_pid  and common_pid  != for_pid ):
 		return

tools/perf/scripts/python/futex-contention.py +2 -2

index 11e70a3..0f5cf43 100644
@@ -21,7 +21,7 @@ thread_blocktime = {}
 lock_waits = {} # long-lived stats on (tid,lock) blockage elapsed time
 process_names = {} # long-lived pid-to-execname mapping
 
-def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm,
+def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm, callchain,
 			      nr, uaddr, op, val, utime, uaddr2, val3):
 	cmd = op & FUTEX_CMD_MASK
 	if cmd != FUTEX_WAIT:

@@ -31,7 +31,7 @@ def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm,
 	thread_thislock[tid] = uaddr
 	thread_blocktime[tid] = nsecs(s, ns)
 
-def syscalls__sys_exit_futex(event, ctxt, cpu, s, ns, tid, comm,
+def syscalls__sys_exit_futex(event, ctxt, cpu, s, ns, tid, comm, callchain,
 			     nr, ret):
 	if thread_blocktime.has_key(tid):
 		elapsed = nsecs(s, ns) - thread_blocktime[tid]

tools/perf/scripts/python/net_dropmonitor.py +1 -1

index b574059..0b6ce8c 100755
@@ -66,7 +66,7 @@ def trace_end():
 	print_drop_table()
 
 # called from perf, when it finds a correspoinding event
-def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm,
+def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm, callchain,
 		   skbaddr, location, protocol):
 	slocation = str(location)
 	try:

tools/perf/scripts/python/netdev-times.py +13 -13

index 9aa0a32..4d21ef2 100644
@@ -224,75 +224,75 @@ def trace_end():
 			(len(rx_skb_list), of_count_rx_skb_list)
 
 # called from perf, when it finds a correspoinding event
-def irq__softirq_entry(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_entry(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
-def irq__softirq_exit(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_exit(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
-def irq__softirq_raise(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_raise(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
 def irq__irq_handler_entry(name, context, cpu, sec, nsec, pid, comm,
-			irq, irq_name):
+			callchain, irq, irq_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			irq, irq_name)
 	all_event_list.append(event_info)
 
-def irq__irq_handler_exit(name, context, cpu, sec, nsec, pid, comm, irq, ret):
+def irq__irq_handler_exit(name, context, cpu, sec, nsec, pid, comm, callchain, irq, ret):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, irq, ret)
 	all_event_list.append(event_info)
 
-def napi__napi_poll(name, context, cpu, sec, nsec, pid, comm, napi, dev_name):
+def napi__napi_poll(name, context, cpu, sec, nsec, pid, comm, callchain, napi, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			napi, dev_name)
 	all_event_list.append(event_info)
 
-def net__netif_receive_skb(name, context, cpu, sec, nsec, pid, comm, skbaddr,
+def net__netif_receive_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr,
 			skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__netif_rx(name, context, cpu, sec, nsec, pid, comm, skbaddr,
+def net__netif_rx(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr,
 			skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__net_dev_queue(name, context, cpu, sec, nsec, pid, comm,
+def net__net_dev_queue(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__net_dev_xmit(name, context, cpu, sec, nsec, pid, comm,
+def net__net_dev_xmit(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, skblen, rc, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, rc ,dev_name)
 	all_event_list.append(event_info)
 
-def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm,
+def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, protocol, location):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, protocol, location)
 	all_event_list.append(event_info)
 
-def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, skbaddr):
+def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr)
 	all_event_list.append(event_info)
 
-def skb__skb_copy_datagram_iovec(name, context, cpu, sec, nsec, pid, comm,
+def skb__skb_copy_datagram_iovec(name, context, cpu, sec, nsec, pid, comm, callchain,
 	skbaddr, skblen):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen)

tools/perf/scripts/python/sched-migration.py +20 -21

index 74d55ec..de66cb3 100644
@@ -369,93 +369,92 @@ def trace_end():
 
 def sched__sched_stat_runtime(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, runtime, vruntime):
+	common_callchain, comm, pid, runtime, vruntime):
 	pass
 
 def sched__sched_stat_iowait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_stat_sleep(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_stat_wait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_process_fork(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	parent_comm, parent_pid, child_comm, child_pid):
+	common_callchain, parent_comm, parent_pid, child_comm, child_pid):
 	pass
 
 def sched__sched_process_wait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_process_exit(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_process_free(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_migrate_task(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, orig_cpu,
+	common_callchain, comm, pid, prio, orig_cpu,
 	dest_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.migrate(headers, pid, prio, orig_cpu, dest_cpu)
 
 def sched__sched_switch(event_name, context, common_cpu,
-	common_secs, common_nsecs, common_pid, common_comm,
+	common_secs, common_nsecs, common_pid, common_comm, common_callchain,
 	prev_comm, prev_pid, prev_prio, prev_state,
 	next_comm, next_pid, next_prio):
 
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.sched_switch(headers, prev_comm, prev_pid, prev_prio, prev_state,
 			 next_comm, next_pid, next_prio)
 
 def sched__sched_wakeup_new(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, success,
+	common_callchain, comm, pid, prio, success,
 	target_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.wake_up(headers, comm, pid, success, target_cpu, 1)
 
 def sched__sched_wakeup(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, success,
+	common_callchain, comm, pid, prio, success,
 	target_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.wake_up(headers, comm, pid, success, target_cpu, 0)
 
 def sched__sched_wait_task(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_kthread_stop_ret(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	ret):
+	common_callchain, ret):
 	pass
 
 def sched__sched_kthread_stop(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid):
+	common_callchain, comm, pid):
 	pass
 
-def trace_unhandled(event_name, context, common_cpu, common_secs, common_nsecs,
-		common_pid, common_comm):
+def trace_unhandled(event_name, context, event_fields_dict):
 	pass

tools/perf/scripts/python/sctop.py +1 -1

index 42c267e..df65a30 100644
@@ -44,7 +44,7 @@ def trace_begin():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 	if for_comm is not None:
 		if common_comm != for_comm:
 			return

tools/perf/scripts/python/syscall-counts-by-pid.py +1 -1

index c64d1c5..9b07b39 100644
@@ -38,7 +38,7 @@ def trace_end():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 
 	if (for_comm and common_comm != for_comm) or \
 	   (for_pid  and common_pid  != for_pid ):

tools/perf/scripts/python/syscall-counts.py +1 -1

index b435d3f..5899ce5 100644
@@ -35,7 +35,7 @@ def trace_end():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 	if for_comm is not None:
 		if common_comm != for_comm:
 			return

tools/perf/util/scripting-engines/trace-event-python.c +102 -4

index 8454dc9..4f81dc7 100644
@@ -32,6 +32,7 @@
 #include "../event.h"
 #include "../thread.h"
 #include "../trace-event.h"
+#include "../machine.h"
 
 PyMODINIT_FUNC initperf_trace_context(void);
 

@@ -236,12 +237,90 @@ static inline struct event_format *find_cache_event(struct perf_evsel *evsel)
 	return event;
 }
 
+
+static PyObject *python_process_callchain(struct perf_sample *sample,
+					 struct perf_evsel *evsel,
+					 struct addr_location *al)
+{
+	PyObject *pylist;
+
+	pylist = PyList_New(0);
+	if (!pylist)
+		Py_FatalError("couldn't create Python list");
+
+	if (!symbol_conf.use_callchain || !sample->callchain)
+		goto exit;
+
+	if (machine__resolve_callchain(al->machine, evsel, al->thread,
+					   sample, NULL, NULL,
+					   PERF_MAX_STACK_DEPTH) != 0) {
+		pr_err("Failed to resolve callchain. Skipping\n");
+		goto exit;
+	}
+	callchain_cursor_commit(&callchain_cursor);
+
+
+	while (1) {
+		PyObject *pyelem;
+		struct callchain_cursor_node *node;
+		node = callchain_cursor_current(&callchain_cursor);
+		if (!node)
+			break;
+
+		pyelem = PyDict_New();
+		if (!pyelem)
+			Py_FatalError("couldn't create Python dictionary");
+
+
+		pydict_set_item_string_decref(pyelem, "ip",
+				PyLong_FromUnsignedLongLong(node->ip));
+
+		if (node->sym) {
+			PyObject *pysym  = PyDict_New();
+			if (!pysym)
+				Py_FatalError("couldn't create Python dictionary");
+			pydict_set_item_string_decref(pysym, "start",
+					PyLong_FromUnsignedLongLong(node->sym->start));
+			pydict_set_item_string_decref(pysym, "end",
+					PyLong_FromUnsignedLongLong(node->sym->end));
+			pydict_set_item_string_decref(pysym, "binding",
+					PyInt_FromLong(node->sym->binding));
+			pydict_set_item_string_decref(pysym, "name",
+					PyString_FromStringAndSize(node->sym->name,
+							node->sym->namelen));
+			pydict_set_item_string_decref(pyelem, "sym", pysym);
+		}
+
+		if (node->map) {
+			struct map *map = node->map;
+			const char *dsoname = "[unknown]";
+			if (map && map->dso && (map->dso->name || map->dso->long_name)) {
+				if (symbol_conf.show_kernel_path && map->dso->long_name)
+					dsoname = map->dso->long_name;
+				else if (map->dso->name)
+					dsoname = map->dso->name;
+			}
+			pydict_set_item_string_decref(pyelem, "dso",
+					PyString_FromString(dsoname));
+		}
+
+		callchain_cursor_advance(&callchain_cursor);
+		PyList_Append(pylist, pyelem);
+		Py_DECREF(pyelem);
+	}
+
+exit:
+	return pylist;
+}
+
+
 static void python_process_tracepoint(struct perf_sample *sample,
 				      struct perf_evsel *evsel,
 				      struct thread *thread,
 				      struct addr_location *al)
 {
-	PyObject *handler, *retval, *context, *t, *obj, *dict = NULL;
+	PyObject *handler, *retval, *context, *t, *obj, *callchain;
+	PyObject *dict = NULL;
 	static char handler_name[256];
 	struct format_field *field;
 	unsigned long long val;

@@ -285,18 +364,23 @@ static void python_process_tracepoint(struct perf_sample *sample,
 	PyTuple_SetItem(t, n++, PyString_FromString(handler_name));
 	PyTuple_SetItem(t, n++, context);
 
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+
 	if (handler) {
 		PyTuple_SetItem(t, n++, PyInt_FromLong(cpu));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(s));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(ns));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(pid));
 		PyTuple_SetItem(t, n++, PyString_FromString(comm));
+		PyTuple_SetItem(t, n++, callchain);
 	} else {
 		pydict_set_item_string_decref(dict, "common_cpu", PyInt_FromLong(cpu));
 		pydict_set_item_string_decref(dict, "common_s", PyInt_FromLong(s));
 		pydict_set_item_string_decref(dict, "common_ns", PyInt_FromLong(ns));
 		pydict_set_item_string_decref(dict, "common_pid", PyInt_FromLong(pid));
 		pydict_set_item_string_decref(dict, "common_comm", PyString_FromString(comm));
+		pydict_set_item_string_decref(dict, "common_callchain", callchain);
 	}
 	for (field = event->format.fields; field; field = field->next) {
 		if (field->flags & FIELD_IS_STRING) {

@@ -329,6 +413,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			pydict_set_item_string_decref(dict, field->name, obj);
 
 	}
+
 	if (!handler)
 		PyTuple_SetItem(t, n++, dict);
 

@@ -360,7 +445,7 @@ static void python_process_general_event(struct perf_sample *sample,
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict;
+	PyObject *handler, *retval, *t, *dict, *callchain;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -400,6 +485,10 @@ static void python_process_general_event(struct perf_sample *sample,
 			PyString_FromString(al->sym->name));
 	}
 
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	pydict_set_item_string_decref(dict, "callchain", callchain);
+
 	PyTuple_SetItem(t, n++, dict);
 	if (_PyTuple_Resize(&t, n) == -1)
 		Py_FatalError("error resizing Python tuple");

@@ -596,6 +685,7 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 		fprintf(ofp, "common_nsecs, ");
 		fprintf(ofp, "common_pid, ");
 		fprintf(ofp, "common_comm,\n\t");
+		fprintf(ofp, "common_callchain, ");
 
 		not_first = 0;
 		count = 0;

@@ -638,7 +728,7 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 				fprintf(ofp, "%%u");
 		}
 
-		fprintf(ofp, "\\n\" %% \\\n\t\t(");
+		fprintf(ofp, "\" %% \\\n\t\t(");
 
 		not_first = 0;
 		count = 0;

@@ -674,7 +764,15 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 				fprintf(ofp, "%s", f->name);
 		}
 
-		fprintf(ofp, "),\n\n");
+		fprintf(ofp, ")\n\n");
+
+		fprintf(ofp, "\t\tfor node in common_callchain:");
+		fprintf(ofp, "\n\t\t\tif 'sym' in node:");
+		fprintf(ofp, "\n\t\t\t\tprint \"\\t[%%x] %%s\" %% (node['ip'], node['sym']['name'])");
+		fprintf(ofp, "\n\t\t\telse:");
+		fprintf(ofp, "\n\t\t\t\tprint \"\t[%%x]\" %% (node['ip'])\n\n");
+		fprintf(ofp, "\t\tprint \"\\n\"\n\n");
+
 	}
 
 	fprintf(ofp, "def trace_unhandled(event_name, context, "

Add callchain to generic and tracepoint events. Since this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

[PATCH 1/3] perf script: Add missing calls to Py_DECREF for return values

Signed-off-by Joseph Schuchart <[email protected]>

---
 tools/perf/util/scripting-engines/trace-event-python.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

tools/perf/util/scripting-engines/trace-event-python.c +10 -2

index 1c41932..631aa85 100644
@@ -50,10 +50,14 @@ static int zero_flag_atom;
 
 static PyObject *main_module, *main_dict;
 
+static void handler_call_die(const char *handler_name) NORETURN;
 static void handler_call_die(const char *handler_name)
 {
 	PyErr_Print();
 	Py_FatalError("problem in Python trace event handler");
+	// Py_FatalError does not return
+	// but we have to make the compiler happy
+	abort();
 }
 
 /*

@@ -97,6 +101,7 @@ static void define_value(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -143,6 +148,7 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -334,6 +340,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {

@@ -341,6 +348,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}

@@ -400,6 +408,7 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);

@@ -521,8 +530,7 @@ static int python_stop_script(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_end");
-	else
-		Py_DECREF(retval);
+	Py_DECREF(retval);
 out:
 	Py_XDECREF(main_dict);
 	Py_XDECREF(main_module);

[PATCH 2/3] perf script: Add callchain to generic and tracepoint events. Since

this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

---
 .../python/Perf-Trace-Util/lib/Perf/Trace/Core.py  |   3 +-
 tools/perf/scripts/python/check-perf-trace.py      |   4 +-
 .../perf/scripts/python/failed-syscalls-by-pid.py  |   2 +-
 tools/perf/scripts/python/futex-contention.py      |   4 +-
 tools/perf/scripts/python/net_dropmonitor.py       |   2 +-
 tools/perf/scripts/python/netdev-times.py          |  26 ++---
 tools/perf/scripts/python/sched-migration.py       |  41 ++++----
 tools/perf/scripts/python/sctop.py                 |   2 +-
 tools/perf/scripts/python/syscall-counts-by-pid.py |   2 +-
 tools/perf/scripts/python/syscall-counts.py        |   2 +-
 .../util/scripting-engines/trace-event-python.c    | 106 ++++++++++++++++++++-
 11 files changed, 146 insertions(+), 48 deletions(-)

tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py +2 -1

index de7211e..38dfb72 100644
@@ -107,12 +107,13 @@ def taskState(state):
 
 class EventHeaders:
 	def __init__(self, common_cpu, common_secs, common_nsecs,
-		     common_pid, common_comm):
+		     common_pid, common_comm, common_callchain):
 		self.cpu = common_cpu
 		self.secs = common_secs
 		self.nsecs = common_nsecs
 		self.pid = common_pid
 		self.comm = common_comm
+		self.callchain = common_callchain
 
 	def ts(self):
 		return (self.secs * (10 ** 9)) + self.nsecs

tools/perf/scripts/python/check-perf-trace.py +2 -2

index 4647a76..334599c 100644
@@ -27,7 +27,7 @@ def trace_end():
 
 def irq__softirq_entry(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	vec):
+	common_callchain, vec):
 		print_header(event_name, common_cpu, common_secs, common_nsecs,
 			common_pid, common_comm)
 

@@ -38,7 +38,7 @@ def irq__softirq_entry(event_name, context, common_cpu,
 
 def kmem__kmalloc(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	call_site, ptr, bytes_req, bytes_alloc,
+	common_callchain, call_site, ptr, bytes_req, bytes_alloc,
 	gfp_flags):
 		print_header(event_name, common_cpu, common_secs, common_nsecs,
 			common_pid, common_comm)

tools/perf/scripts/python/failed-syscalls-by-pid.py +1 -1

index 266a836..cafeff3 100644
@@ -39,7 +39,7 @@ def trace_end():
 
 def raw_syscalls__sys_exit(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, ret):
+	common_callchain, id, ret):
 	if (for_comm and common_comm != for_comm) or \
 	   (for_pid  and common_pid  != for_pid ):
 		return

tools/perf/scripts/python/futex-contention.py +2 -2

index 11e70a3..0f5cf43 100644
@@ -21,7 +21,7 @@ thread_blocktime = {}
 lock_waits = {} # long-lived stats on (tid,lock) blockage elapsed time
 process_names = {} # long-lived pid-to-execname mapping
 
-def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm,
+def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm, callchain,
 			      nr, uaddr, op, val, utime, uaddr2, val3):
 	cmd = op & FUTEX_CMD_MASK
 	if cmd != FUTEX_WAIT:

@@ -31,7 +31,7 @@ def syscalls__sys_enter_futex(event, ctxt, cpu, s, ns, tid, comm,
 	thread_thislock[tid] = uaddr
 	thread_blocktime[tid] = nsecs(s, ns)
 
-def syscalls__sys_exit_futex(event, ctxt, cpu, s, ns, tid, comm,
+def syscalls__sys_exit_futex(event, ctxt, cpu, s, ns, tid, comm, callchain,
 			     nr, ret):
 	if thread_blocktime.has_key(tid):
 		elapsed = nsecs(s, ns) - thread_blocktime[tid]

tools/perf/scripts/python/net_dropmonitor.py +1 -1

index b574059..0b6ce8c 100755
@@ -66,7 +66,7 @@ def trace_end():
 	print_drop_table()
 
 # called from perf, when it finds a correspoinding event
-def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm,
+def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm, callchain,
 		   skbaddr, location, protocol):
 	slocation = str(location)
 	try:

tools/perf/scripts/python/netdev-times.py +13 -13

index 9aa0a32..4d21ef2 100644
@@ -224,75 +224,75 @@ def trace_end():
 			(len(rx_skb_list), of_count_rx_skb_list)
 
 # called from perf, when it finds a correspoinding event
-def irq__softirq_entry(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_entry(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
-def irq__softirq_exit(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_exit(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
-def irq__softirq_raise(name, context, cpu, sec, nsec, pid, comm, vec):
+def irq__softirq_raise(name, context, cpu, sec, nsec, pid, comm, callchain, vec):
 	if symbol_str("irq__softirq_entry", "vec", vec) != "NET_RX":
 		return
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, vec)
 	all_event_list.append(event_info)
 
 def irq__irq_handler_entry(name, context, cpu, sec, nsec, pid, comm,
-			irq, irq_name):
+			callchain, irq, irq_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			irq, irq_name)
 	all_event_list.append(event_info)
 
-def irq__irq_handler_exit(name, context, cpu, sec, nsec, pid, comm, irq, ret):
+def irq__irq_handler_exit(name, context, cpu, sec, nsec, pid, comm, callchain, irq, ret):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, irq, ret)
 	all_event_list.append(event_info)
 
-def napi__napi_poll(name, context, cpu, sec, nsec, pid, comm, napi, dev_name):
+def napi__napi_poll(name, context, cpu, sec, nsec, pid, comm, callchain, napi, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			napi, dev_name)
 	all_event_list.append(event_info)
 
-def net__netif_receive_skb(name, context, cpu, sec, nsec, pid, comm, skbaddr,
+def net__netif_receive_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr,
 			skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__netif_rx(name, context, cpu, sec, nsec, pid, comm, skbaddr,
+def net__netif_rx(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr,
 			skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__net_dev_queue(name, context, cpu, sec, nsec, pid, comm,
+def net__net_dev_queue(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, skblen, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, dev_name)
 	all_event_list.append(event_info)
 
-def net__net_dev_xmit(name, context, cpu, sec, nsec, pid, comm,
+def net__net_dev_xmit(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, skblen, rc, dev_name):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen, rc ,dev_name)
 	all_event_list.append(event_info)
 
-def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm,
+def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm, callchain,
 			skbaddr, protocol, location):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, protocol, location)
 	all_event_list.append(event_info)
 
-def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, skbaddr):
+def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr)
 	all_event_list.append(event_info)
 
-def skb__skb_copy_datagram_iovec(name, context, cpu, sec, nsec, pid, comm,
+def skb__skb_copy_datagram_iovec(name, context, cpu, sec, nsec, pid, comm, callchain,
 	skbaddr, skblen):
 	event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm,
 			skbaddr, skblen)

tools/perf/scripts/python/sched-migration.py +20 -21

index 74d55ec..de66cb3 100644
@@ -369,93 +369,92 @@ def trace_end():
 
 def sched__sched_stat_runtime(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, runtime, vruntime):
+	common_callchain, comm, pid, runtime, vruntime):
 	pass
 
 def sched__sched_stat_iowait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_stat_sleep(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_stat_wait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, delay):
+	common_callchain, comm, pid, delay):
 	pass
 
 def sched__sched_process_fork(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	parent_comm, parent_pid, child_comm, child_pid):
+	common_callchain, parent_comm, parent_pid, child_comm, child_pid):
 	pass
 
 def sched__sched_process_wait(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_process_exit(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_process_free(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_migrate_task(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, orig_cpu,
+	common_callchain, comm, pid, prio, orig_cpu,
 	dest_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.migrate(headers, pid, prio, orig_cpu, dest_cpu)
 
 def sched__sched_switch(event_name, context, common_cpu,
-	common_secs, common_nsecs, common_pid, common_comm,
+	common_secs, common_nsecs, common_pid, common_comm, common_callchain,
 	prev_comm, prev_pid, prev_prio, prev_state,
 	next_comm, next_pid, next_prio):
 
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.sched_switch(headers, prev_comm, prev_pid, prev_prio, prev_state,
 			 next_comm, next_pid, next_prio)
 
 def sched__sched_wakeup_new(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, success,
+	common_callchain, comm, pid, prio, success,
 	target_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.wake_up(headers, comm, pid, success, target_cpu, 1)
 
 def sched__sched_wakeup(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio, success,
+	common_callchain, comm, pid, prio, success,
 	target_cpu):
 	headers = EventHeaders(common_cpu, common_secs, common_nsecs,
-				common_pid, common_comm)
+				common_pid, common_comm, common_callchain)
 	parser.wake_up(headers, comm, pid, success, target_cpu, 0)
 
 def sched__sched_wait_task(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid, prio):
+	common_callchain, comm, pid, prio):
 	pass
 
 def sched__sched_kthread_stop_ret(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	ret):
+	common_callchain, ret):
 	pass
 
 def sched__sched_kthread_stop(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	comm, pid):
+	common_callchain, comm, pid):
 	pass
 
-def trace_unhandled(event_name, context, common_cpu, common_secs, common_nsecs,
-		common_pid, common_comm):
+def trace_unhandled(event_name, context, event_fields_dict):
 	pass

tools/perf/scripts/python/sctop.py +1 -1

index c9f3058..61621b9 100644
@@ -44,7 +44,7 @@ def trace_begin():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 	if for_comm is not None:
 		if common_comm != for_comm:
 			return

tools/perf/scripts/python/syscall-counts-by-pid.py +1 -1

index cf2054c..daf314c 100644
@@ -38,7 +38,7 @@ def trace_end():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 
 	if (for_comm and common_comm != for_comm) or \
 	   (for_pid  and common_pid  != for_pid ):

tools/perf/scripts/python/syscall-counts.py +1 -1

index 92b2938..e66a773 100644
@@ -35,7 +35,7 @@ def trace_end():
 
 def raw_syscalls__sys_enter(event_name, context, common_cpu,
 	common_secs, common_nsecs, common_pid, common_comm,
-	id, args):
+	common_callchain, id, args):
 	if for_comm is not None:
 		if common_comm != for_comm:
 			return

tools/perf/util/scripting-engines/trace-event-python.c +102 -4

index 631aa85..dae78e4 100644
@@ -32,6 +32,7 @@
 #include "../event.h"
 #include "../thread.h"
 #include "../trace-event.h"
+#include "../machine.h"
 
 PyMODINIT_FUNC initperf_trace_context(void);
 

@@ -237,12 +238,90 @@ static inline struct event_format *find_cache_event(struct perf_evsel *evsel)
 	return event;
 }
 
+
+static PyObject *python_process_callchain(struct perf_sample *sample,
+					 struct perf_evsel *evsel,
+					 struct addr_location *al)
+{
+	PyObject *pylist;
+
+	pylist = PyList_New(0);
+	if (!pylist)
+		Py_FatalError("couldn't create Python list");
+
+	if (!symbol_conf.use_callchain || !sample->callchain)
+		goto exit;
+
+	if (machine__resolve_callchain(al->machine, evsel, al->thread,
+					   sample, NULL, NULL,
+					   PERF_MAX_STACK_DEPTH) != 0) {
+		pr_err("Failed to resolve callchain. Skipping\n");
+		goto exit;
+	}
+	callchain_cursor_commit(&callchain_cursor);
+
+
+	while (1) {
+		PyObject *pyelem;
+		struct callchain_cursor_node *node;
+		node = callchain_cursor_current(&callchain_cursor);
+		if (!node)
+			break;
+
+		pyelem = PyDict_New();
+		if (!pyelem)
+			Py_FatalError("couldn't create Python dictionary");
+
+
+		pydict_set_item_string_decref(pyelem, "ip",
+				PyLong_FromUnsignedLongLong(node->ip));
+
+		if (node->sym) {
+			PyObject *pysym  = PyDict_New();
+			if (!pysym)
+				Py_FatalError("couldn't create Python dictionary");
+			pydict_set_item_string_decref(pysym, "start",
+					PyLong_FromUnsignedLongLong(node->sym->start));
+			pydict_set_item_string_decref(pysym, "end",
+					PyLong_FromUnsignedLongLong(node->sym->end));
+			pydict_set_item_string_decref(pysym, "binding",
+					PyInt_FromLong(node->sym->binding));
+			pydict_set_item_string_decref(pysym, "name",
+					PyString_FromStringAndSize(node->sym->name,
+							node->sym->namelen));
+			pydict_set_item_string_decref(pyelem, "sym", pysym);
+		}
+
+		if (node->map) {
+			struct map *map = node->map;
+			const char *dsoname = "[unknown]";
+			if (map && map->dso && (map->dso->name || map->dso->long_name)) {
+				if (symbol_conf.show_kernel_path && map->dso->long_name)
+					dsoname = map->dso->long_name;
+				else if (map->dso->name)
+					dsoname = map->dso->name;
+			}
+			pydict_set_item_string_decref(pyelem, "dso",
+					PyString_FromString(dsoname));
+		}
+
+		callchain_cursor_advance(&callchain_cursor);
+		PyList_Append(pylist, pyelem);
+		Py_DECREF(pyelem);
+	}
+
+exit:
+	return pylist;
+}
+
+
 static void python_process_tracepoint(struct perf_sample *sample,
 				      struct perf_evsel *evsel,
 				      struct thread *thread,
 				      struct addr_location *al)
 {
-	PyObject *handler, *retval, *context, *t, *obj, *dict = NULL;
+	PyObject *handler, *retval, *context, *t, *obj, *callchain;
+	PyObject *dict = NULL;
 	static char handler_name[256];
 	struct format_field *field;
 	unsigned long long val;

@@ -286,18 +365,23 @@ static void python_process_tracepoint(struct perf_sample *sample,
 	PyTuple_SetItem(t, n++, PyString_FromString(handler_name));
 	PyTuple_SetItem(t, n++, context);
 
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+
 	if (handler) {
 		PyTuple_SetItem(t, n++, PyInt_FromLong(cpu));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(s));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(ns));
 		PyTuple_SetItem(t, n++, PyInt_FromLong(pid));
 		PyTuple_SetItem(t, n++, PyString_FromString(comm));
+		PyTuple_SetItem(t, n++, callchain);
 	} else {
 		pydict_set_item_string_decref(dict, "common_cpu", PyInt_FromLong(cpu));
 		pydict_set_item_string_decref(dict, "common_s", PyInt_FromLong(s));
 		pydict_set_item_string_decref(dict, "common_ns", PyInt_FromLong(ns));
 		pydict_set_item_string_decref(dict, "common_pid", PyInt_FromLong(pid));
 		pydict_set_item_string_decref(dict, "common_comm", PyString_FromString(comm));
+		pydict_set_item_string_decref(dict, "common_callchain", callchain);
 	}
 	for (field = event->format.fields; field; field = field->next) {
 		if (field->flags & FIELD_IS_STRING) {

@@ -330,6 +414,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			pydict_set_item_string_decref(dict, field->name, obj);
 
 	}
+
 	if (!handler)
 		PyTuple_SetItem(t, n++, dict);
 

@@ -361,7 +446,7 @@ static void python_process_general_event(struct perf_sample *sample,
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict;
+	PyObject *handler, *retval, *t, *dict, *callchain;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -401,6 +486,10 @@ static void python_process_general_event(struct perf_sample *sample,
 			PyString_FromString(al->sym->name));
 	}
 
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	pydict_set_item_string_decref(dict, "callchain", callchain);
+
 	PyTuple_SetItem(t, n++, dict);
 	if (_PyTuple_Resize(&t, n) == -1)
 		Py_FatalError("error resizing Python tuple");

@@ -597,6 +686,7 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 		fprintf(ofp, "common_nsecs, ");
 		fprintf(ofp, "common_pid, ");
 		fprintf(ofp, "common_comm,\n\t");
+		fprintf(ofp, "common_callchain, ");
 
 		not_first = 0;
 		count = 0;

@@ -640,7 +730,7 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 				fprintf(ofp, "%%u");
 		}
 
-		fprintf(ofp, "\\n\" %% \\\n\t\t(");
+		fprintf(ofp, "\" %% \\\n\t\t(");
 
 		not_first = 0;
 		count = 0;

@@ -676,7 +766,15 @@ static int python_generate_script(struct pevent *pevent, const char *outfile)
 				fprintf(ofp, "%s", f->name);
 		}
 
-		fprintf(ofp, "),\n\n");
+		fprintf(ofp, ")\n\n");
+
+		fprintf(ofp, "\t\tfor node in common_callchain:");
+		fprintf(ofp, "\n\t\t\tif 'sym' in node:");
+		fprintf(ofp, "\n\t\t\t\tprint \"\\t[%%x] %%s\" %% (node['ip'], node['sym']['name'])");
+		fprintf(ofp, "\n\t\t\telse:");
+		fprintf(ofp, "\n\t\t\t\tprint \"\t[%%x]\" %% (node['ip'])\n\n");
+		fprintf(ofp, "\t\tprint \"\\n\"\n\n");
+
 	}
 
 	fprintf(ofp, "def trace_unhandled(event_name, context, "

[PATCH 2/3] perf script: Add callchain to generic and tracepoint events. Since
this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

Quoted 3 lines · Joseph Schuchart

[PATCH 2/3] perf script: Add callchain to generic and tracepoint events. Since
this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

still having issues to apply 2nd patch on tip/perf/core (8b5b584)

Applying: perf script: Add missing calls to Py_DECREF for return values
Applying: perf script: Add callchain to generic and tracepoint events
error: patch failed: tools/perf/util/scripting-engines/trace-event-python.c:237
error: tools/perf/util/scripting-engines/trace-event-python.c: patch does not apply Patch failed at 0002 perf script: Add callchain to generic and tracepoint events The copy of the patch that failed is found in:
/home/jolsa/kernel.org/linux-perf/.git/rebase-apply/patch
When you have resolved this problem, run "git am --resolved".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".

thanks,
jirka

Jirka,

I tried to apply the patches to the latest version of the perf/core branch before sending. Can you please send me some more information about the conflict?

Quoted 3 lines · Joseph Schuchart

Jirka,

I tried to apply the patches to the latest version of the perf/core branch before sending. Can you please send me some more information about the conflict?

is your HEAD 8b5b584 ?

[jolsa@krava perf]$ git show --oneline tip/perf/core
8b5b584 Merge tag 'perf-core-for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf into perf/core

jirka

perf script: Add missing calls to Py_DECREF for return values

Signed-off-by Joseph Schuchart <[email protected]>

---
 tools/perf/util/scripting-engines/trace-event-python.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

tools/perf/util/scripting-engines/trace-event-python.c +10 -2

index e55b65a..b6c1a69 100644
@@ -50,10 +50,14 @@ static int zero_flag_atom;
 
 static PyObject *main_module, *main_dict;
 
+static void handler_call_die(const char *handler_name) NORETURN;
 static void handler_call_die(const char *handler_name)
 {
 	PyErr_Print();
 	Py_FatalError("problem in Python trace event handler");
+	// Py_FatalError does not return
+	// but we have to make the compiler happy
+	abort();
 }
 
 /*

@@ -97,6 +101,7 @@ static void define_value(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -143,6 +148,7 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -361,6 +367,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {

@@ -368,6 +375,7 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}

@@ -427,6 +435,7 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);

@@ -548,8 +557,7 @@ static int python_stop_script(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_end");
-	else
-		Py_DECREF(retval);
+	Py_DECREF(retval);
 out:
 	Py_XDECREF(main_dict);
 	Py_XDECREF(main_module);

Signature · 1 line

1.8.1.2

perf script: Add callchain to generic and tracepoint events. Since this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

Em Wed, Jul 09, 2014 at 04:16:33PM +0200, Joseph Schuchart escreveu:

Quoted 7 lines

perf script: Add callchain to generic and tracepoint events. Since this alters the interface for the python scripts, also adjust the script generation and the provided scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

This one is not applying, I'm checking the reason for its malformation as stated in:

[acme@ssdandy linux]$ patch -p1 < /wb/1.patch
patching file tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py
Hunk #1 FAILED at 107.
1 out of 1 hunk FAILED -- saving rejects to file tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py.rej patching file tools/perf/scripts/python/check-perf-trace.py
Hunk #1 succeeded at 27 with fuzz 1.
Hunk #2 succeeded at 37 with fuzz 2 (offset -1 lines).
patching file tools/perf/scripts/python/failed-syscalls-by-pid.py
Hunk #1 FAILED at 39.
1 out of 1 hunk FAILED -- saving rejects to file tools/perf/scripts/python/failed-syscalls-by-pid.py.rej patching file tools/perf/scripts/python/futex-contention.py
patch: **** malformed patch at line 154: ain,

Em Wed, Jul 09, 2014 at 04:09:21PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 5 lines

Em Wed, Jul 09, 2014 at 04:16:33PM +0200, Joseph Schuchart escreveu:

This one is not applying, I'm checking the reason for its malformation as stated in:

Quoted 12 lines

[acme@ssdandy linux]$ patch -p1 < /wb/1.patch
patching file tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py
Hunk #1 FAILED at 107.
1 out of 1 hunk FAILED -- saving rejects to file tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py.rej patching file tools/perf/scripts/python/check-perf-trace.py
Hunk #1 succeeded at 27 with fuzz 1.
Hunk #2 succeeded at 37 with fuzz 2 (offset -1 lines).
patching file tools/perf/scripts/python/failed-syscalls-by-pid.py
Hunk #1 FAILED at 39.
1 out of 1 hunk FAILED -- saving rejects to file tools/perf/scripts/python/failed-syscalls-by-pid.py.rej patching file tools/perf/scripts/python/futex-contention.py
patch: **** malformed patch at line 154: ain,

It is full of:

self.cpu =3D common_cpu

Em Wed, Jul 09, 2014 at 04:12:23PM -0300, Arnaldo Carvalho de Melo escreveu:

Quoted 5 lines

Em Wed, Jul 09, 2014 at 04:09:21PM -0300, Arnaldo Carvalho de Melo escreveu:

It is full of:

self.cpu =3D common_cpu

Ok, I applied the first patch on this series, please see my perf/core branch at:

git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

You can have it with:

git remote add acme git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git
git checkout -b perf/core acme/perf/core

Then please create the patches and follow the instructions about Thunderbird at:

https://www.kernel.org/pub/software/scm/git/docs/git-format-patch.html

So that it doesn't mangles the patch and resend, ok?

- Arnaldo

[PATCH 3/3] perf script: Provide additional sample information on generic events to Python scripts, including pid, tid, and cpu for which the event was recorded. At the moment, the pointer to the sample struct is passed to scripts, which seems to be of little use. The patch puts this information in dictionaries for easy access by Python scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

---
 .../util/scripting-engines/trace-event-python.c    | 23 +++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

tools/perf/util/scripting-engines/trace-event-python.c +20 -3

index dae78e4..64923cd 100644
@@ -446,7 +446,7 @@ static void python_process_general_event(struct perf_sample *sample,
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict, *callchain;
+	PyObject *handler, *retval, *t, *dict, *callchain, *dict_sample;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -462,6 +462,10 @@ static void python_process_general_event(struct perf_sample *sample,
 	if (!dict)
 		Py_FatalError("couldn't create Python dictionary");
 
+	dict_sample = PyDict_New();
+	if (!dict_sample)
+		Py_FatalError("couldn't create Python dictionary");
+
 	snprintf(handler_name, sizeof(handler_name), "%s", "process_event");
 
 	handler = PyDict_GetItemString(main_dict, handler_name);

@@ -471,8 +475,21 @@ static void python_process_general_event(struct perf_sample *sample,
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	pydict_set_item_string_decref(dict_sample, "pid",
+			PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid",
+			PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu",
+			PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "ip",
+			PyLong_FromUnsignedLongLong(sample->ip));
+	pydict_set_item_string_decref(dict_sample, "time",
+			PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict_sample, "period",
+			PyLong_FromUnsignedLongLong(sample->period));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",

Hi Joseph,

Sorry for late review, this looks very useful.. But please send a separate email for each patch and make it inlined (not attached) in the next version.

On Thu, 03 Apr 2014 10:57:39 +0200, Joseph Schuchart wrote:

[SNIP]

Quoted 12 lines

static void python_process_tracepoint(struct perf_sample *sample,

(quoted diff) +2 -1

 				      struct perf_evsel *evsel,
 				      struct thread *thread,
 				      struct addr_location *al)
 {
-	PyObject *handler, *retval, *context, *t, *obj, *dict = NULL;
+	PyObject *handler, *retval, *context, *t, *obj, *callchain;
+	PyObject *dict = NULL;
 	static char handler_name[256];
 	struct format_field *field;
 	unsigned long long val;

Review static void python_process_tracepoint(struct perf_sample *sample, · line 415

Quoted 10 lines

(quoted diff) +7 -0

@@ -327,6 +406,14 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			pydict_set_item_string_decref(dict, field->name, obj);
 
 	}
+
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	if (handler)
+		PyTuple_SetItem(t, n++, callchain);
+	else
+		pydict_set_item_string_decref(dict, "callchain", callchain);

But this makes the script not runnable anymore due to argument number mismatch:

  $ perf script -s perf-script.py
  ...
  TypeError: sched__sched_wakeup() takes exactly 12 arguments (13 given)
  Fatal Python error: problem in Python trace event handler
  Aborted (core dumped)

So you need to change to pass callchain when generating the handler.

tools/perf/util/scripting-engines/trace-event-python.c +1 -1

index 255d45177613..94e395c9a875 100644
@@ -716,7 +716,7 @@ static int python_generate_script(struct pevent *pevent, con
 
                        fprintf(ofp, "%s", f->name);
                }
-               fprintf(ofp, "):\n");
+               fprintf(ofp, ", callchain):\n");
 
                fprintf(ofp, "\t\tprint_header(event_name, common_cpu, "
                        "common_secs, common_nsecs,\n\t\t\t"

Also I think it's very useful to generate code to print callchain by default if exists - something like below?

	for node in callchain:
		if 'sym' in node:
			print "\t[%x] %s" % (node['ip'], node['sym']['name'])
		else:
			print "\t[%x]" % (node['ip'])

  $ perf script -s perf-script.py
  in trace_begin
  sched__sched_wakeup      8 5021904.056096444    19713 :19713                comm=perf, pid=19714, prio=120, success=1, target_cpu=9
	[ffffffff81091512] ttwu_do_wakeup
	[ffffffff810916d6] ttwu_do_activate.constprop.87
	[ffffffff81093b64] try_to_wake_up
	[ffffffff81093c22] default_wake_function
	[ffffffff8108348d] autoremove_wake_function
	[ffffffff8108b215] __wake_up_common
	[ffffffff8108e933] __wake_up_sync_key
	[ffffffff811a5b20] pipe_write
	[ffffffff8119cc07] do_sync_write
	[ffffffff8119d2cc] vfs_write
	[ffffffff8119d762] sys_write
	[ffffffff816640d9] system_call_fastpath
  sched__sched_wakeup      8 5021904.056100334    19713 :19713                comm=perf, pid=19714, prio=120, success=1, target_cpu=9
	[ffffffff81091512] ttwu_do_wakeup
	[ffffffff810916d6] ttwu_do_activate.constprop.87
	[ffffffff81093b64] try_to_wake_up
	[ffffffff81093c22] default_wake_function
	[ffffffff8108348d] autoremove_wake_function
	[ffffffff8108b215] __wake_up_common
	[ffffffff8108e933] __wake_up_sync_key
	[ffffffff811a5b20] pipe_write
	[ffffffff8119cc07] do_sync_write
	[ffffffff8119d2cc] vfs_write
	[ffffffff8119d762] sys_write
	[ffffffff816640d9] system_call_fastpath
  ...

Quoted 4 lines

(quoted diff) +1 -0

+
 	if (!handler)
 		PyTuple_SetItem(t, n++, dict);
 

[SNIP]

Quoted 8 lines

(quoted diff) +2 -2

@@ -385,15 +476,30 @@ static void python_process_general_event(struct perf_sample *sample,
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	pydict_set_item_string_decref(dict_sample, "pid",

Review static void python_process_general_event(struct perf_sample *sample, · line 490

Quoted 10 lines

(quoted diff) +10 -0

+			PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid",
+			PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu",
+			PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "time",
+			PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict_sample, "period",
+			PyLong_FromUnsignedLongLong(sample->period));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);

And I think this part should be a separate patch.

Review static void python_process_general_event(struct perf_sample *sample, · line 502

Quoted 13 lines

(quoted diff) +6 -1

+
+	/* ip unwinding */
+	callchain = python_process_callchain(sample, evsel, al);
+	pydict_set_item_string_decref(dict, "callchain", callchain);
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",
 			PyString_FromString(thread__comm_str(thread)));
 	if (al->map) {
 		pydict_set_item_string_decref(dict, "dso",
-			PyString_FromString(al->map->dso->name));
+				PyString_FromString(al->map->dso->name));

It seems like an unnecessary change.

Quoted 20 lines

(quoted diff) +0 -0

 	}
 	if (al->sym) {
 		pydict_set_item_string_decref(dict, "symbol",

Perf: Fix possible memory leaks in Python interface

The function PyObject_CallObject() returns a new PyObject reference on which Py_DECREF has to be called to avoid memory leaks. This patch adds these calls where necessary.

Signed-off-by Joseph Schuchart <[email protected]>

tools/perf/util/scripting-engines/trace-event-python.c +0 -0

index cd9774d..ee17f64 100644

tools/perf/util/scripting-engines/trace-event-python.c static void define_value(enum print_arg_type field_type, · line 101

Quoted 5 lines

tools/perf/util/scripting-engines/trace-event-python.c +2 -0

@@ -97,6 +97,8 @@ static void define_value(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);

It looks like the handler_call_die() is never returned. So we can get rid of the 'else' (like in the last hunk) for simplicity.

Thanks,
Namhyung

Quoted 50 lines

tools/perf/util/scripting-engines/trace-event-python.c +10 -2

 	}
 
 	Py_DECREF(t);

@@ -143,6 +145,8 @@ static void define_field(enum print_arg_type field_type,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	}
 
 	Py_DECREF(t);

@@ -333,6 +337,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 		retval = PyObject_CallObject(handler, t);
 		if (retval == NULL)
 			handler_call_die(handler_name);
+		else
+			Py_DECREF(retval);
 	} else {
 		handler = PyDict_GetItemString(main_dict, "trace_unhandled");
 		if (handler && PyCallable_Check(handler)) {

@@ -340,6 +346,8 @@ static void python_process_tracepoint(struct perf_sample *sample,
 			retval = PyObject_CallObject(handler, t);
 			if (retval == NULL)
 				handler_call_die("trace_unhandled");
+			else
+				Py_DECREF(retval);
 		}
 		Py_DECREF(dict);
 	}

@@ -399,6 +407,8 @@ static void python_process_general_event(struct perf_sample *sample,
 	retval = PyObject_CallObject(handler, t);
 	if (retval == NULL)
 		handler_call_die(handler_name);
+	else
+		Py_DECREF(retval);
 exit:
 	Py_DECREF(dict);
 	Py_DECREF(t);

@@ -444,8 +454,8 @@ static int run_start_sub(void)
 	retval = PyObject_CallObject(handler, NULL);
 	if (retval == NULL)
 		handler_call_die("trace_begin");
-
-	Py_DECREF(retval);
+	else
+		Py_DECREF(retval);
 	return err;
 error:
 	Py_XDECREF(main_dict);

Provide additional sample information on generic events
to Python scripts, including pid, tid, and cpu for which the event was recorded. At the moment, the pointer to the sample struct is passed to scripts, which seems to be of little use. The patch puts this information in a dictionary for easy access by Python scripts.

Signed-off-by Joseph Schuchart <[email protected]>

Acked-by Thomas Ilsche <[email protected]>

---
 .../util/scripting-engines/trace-event-python.c     | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

tools/perf/util/scripting-engines/trace-event-python.c +18 -3

index 4f81dc7..c8d148b 100644
@@ -445,7 +445,7 @@ static void python_process_general_event(struct perf_sample *sample,
 					 struct thread *thread,
 					 struct addr_location *al)
 {
-	PyObject *handler, *retval, *t, *dict, *callchain;
+	PyObject *handler, *retval, *t, *dict, *callchain, *dict_sample;
 	static char handler_name[64];
 	unsigned n = 0;
 

@@ -461,6 +461,10 @@ static void python_process_general_event(struct perf_sample *sample,
 	if (!dict)
 		Py_FatalError("couldn't create Python dictionary");
 
+	dict_sample = PyDict_New();
+	if (!dict_sample)
+		Py_FatalError("couldn't create Python dictionary");
+
 	snprintf(handler_name, sizeof(handler_name), "%s", "process_event");
 
 	handler = PyDict_GetItemString(main_dict, handler_name);

@@ -470,8 +474,19 @@ static void python_process_general_event(struct perf_sample *sample,
 	pydict_set_item_string_decref(dict, "ev_name", PyString_FromString(perf_evsel__name(evsel)));
 	pydict_set_item_string_decref(dict, "attr", PyString_FromStringAndSize(
 			(const char *)&evsel->attr, sizeof(evsel->attr)));
-	pydict_set_item_string_decref(dict, "sample", PyString_FromStringAndSize(
-			(const char *)sample, sizeof(*sample)));
+
+	pydict_set_item_string_decref(dict_sample, "pid",
+			PyInt_FromLong(sample->pid));
+	pydict_set_item_string_decref(dict_sample, "tid",
+			PyInt_FromLong(sample->tid));
+	pydict_set_item_string_decref(dict_sample, "cpu",
+			PyInt_FromLong(sample->cpu));
+	pydict_set_item_string_decref(dict_sample, "time",
+			PyLong_FromUnsignedLongLong(sample->time));
+	pydict_set_item_string_decref(dict_sample, "period",
+			PyLong_FromUnsignedLongLong(sample->period));
+	pydict_set_item_string_decref(dict, "sample", dict_sample);
+
 	pydict_set_item_string_decref(dict, "raw_buf", PyString_FromStringAndSize(
 			(const char *)sample->raw_data, sample->raw_size));
 	pydict_set_item_string_decref(dict, "comm",