Skip to contentSkip to conversation

trace-cruncher: Add Kprobes

PATCH New Yordan Karadzhov (VMware) 1 participant 4 messages Jul 12, 2021

lore
Share & tools
lore
#0 lore raw

v2 cover trace-cruncher: Add Kprobes

This adds to trace-cruncher a basic support for Kprobes.

Changes in v2:
- Fixed naming clash in ftracepy-utils
- Fixed naming clash in tests

Yordan Karadzhov (VMware) (3):
trace-cruncher: Add support for Kprobes
trace-cruncher: Add events to utils
trace-cruncher: Add Kprobe example

 examples/kprobe_open.py               |  44 ++++
 src/ftracepy-utils.c                  | 366 ++++++++++++++++++++++++--
 src/ftracepy-utils.h                  |  30 +++
 src/ftracepy.c                        |  55 ++++
 tests/1_unit/test_01_ftracepy_unit.py |  57 ++++
 tracecruncher/ft_utils.py             | 182 +++++++++++++
 6 files changed, 716 insertions(+), 18 deletions(-)
 create mode 100755 examples/kprobe_open.py
Signature · 1 line
2.27.0
#1 lore raw

PATCH v2 1/3 trace-cruncher: Add support for Kprobes

Kprobes is a built-in debugging mechanism for the Linux kernel, which can be used extract monitoring data from a running production system. Here we add to trace-cruncher a basic support for using Kprobes.

---
 src/ftracepy-utils.c                  | 366 ++++++++++++++++++++++++--
 src/ftracepy-utils.h                  |  30 +++
 src/ftracepy.c                        |  55 ++++
 tests/1_unit/test_01_ftracepy_unit.py |  57 ++++
 4 files changed, 490 insertions(+), 18 deletions(-)
src/ftracepy-utils.c +348 -18
index 91a319e..dfb0669 100644
@@ -100,6 +100,7 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args,
 							  PyObject *kwargs)
 {
 	struct tep_format_field *field;
+	int field_offset, field_size;
 	const char *field_name;
 	PyTepRecord *record;
 
@@ -124,11 +125,24 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args,
 		return NULL;
 	}
 
-	if (!field->size)
+	if (field->flags & TEP_FIELD_IS_DYNAMIC) {
+		unsigned long long val;
+
+		val = tep_read_number(self->ptrObj->tep,
+				      record->ptrObj->data + field->offset,
+				      field->size);
+		field_offset = val & 0xffff;
+		field_size = val >> 16;
+	} else {
+		field_offset = field->offset;
+		field_size = field->size;
+	}
+
+	if (!field_size)
 		return PyUnicode_FromString("(nil)");
 
 	if (field->flags & TEP_FIELD_IS_STRING) {
-		char *val_str = record->ptrObj->data + field->offset;
+		char *val_str = record->ptrObj->data + field_offset;
 		return PyUnicode_FromString(val_str);
 	} else if (is_number(field)) {
 		unsigned long long val;
@@ -136,7 +150,7 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args,
 		tep_read_number_field(field, record->ptrObj->data, &val);
 		return PyLong_FromLong(val);
 	} else if (field->flags & TEP_FIELD_IS_POINTER) {
-		void *val = record->ptrObj->data + field->offset;
+		void *val = record->ptrObj->data + field_offset;
 		char ptr_string[11];
 
 		sprintf(ptr_string, "%p", val);
@@ -1058,14 +1072,33 @@ PyObject *PyFtrace_disable_events(PyObject *self, PyObject *args,
 	Py_RETURN_NONE;
 }
 
+static PyObject *event_is_enabled(struct tracefs_instance *instance,
+				  const char *system, const char *event)
+{
+	char *file, *val;
+	PyObject *ret;
+
+	if (!get_event_enable_file(instance, system, event, &file))
+		return NULL;
+
+	if (read_from_file(instance, file, &val) <= 0)
+		return NULL;
+
+	trim_new_line(val);
+	ret = PyUnicode_FromString(val);
+
+	free(file);
+	free(val);
+
+	return ret;
+}
+
 PyObject *PyFtrace_event_is_enabled(PyObject *self, PyObject *args,
 						    PyObject *kwargs)
 {
 	static char *kwlist[] = {"instance", "system", "event", NULL};
 	const char *instance_name, *system, *event;
 	struct tracefs_instance *instance;
-	char *file, *val;
-	PyObject *ret;
 
 	instance_name = system = event = NO_ARG;
 	if (!PyArg_ParseTupleAndKeywords(args,
@@ -1081,19 +1114,7 @@ PyObject *PyFtrace_event_is_enabled(PyObject *self, PyObject *args,
 	if (!get_optional_instance(instance_name, &instance))
 		return false;
 
-	if (!get_event_enable_file(instance, system, event, &file))
-		return NULL;
-
-	if (read_from_file(instance, file, &val) <= 0)
-		return NULL;
-
-	trim_new_line(val);
-	ret = PyUnicode_FromString(val);
-
-	free(file);
-	free(val);
-
-	return ret;
+	return event_is_enabled(instance, system, event);
 }
 
 PyObject *PyFtrace_set_event_filter(PyObject *self, PyObject *args,
@@ -1472,6 +1493,314 @@ PyObject *PyFtrace_supported_options(PyObject *self, PyObject *args,
 	return get_option_list(instance, false);
 }
 
+static void *kprobe_root = NULL;
+
+static int kprobe_compare(const void *a, const void *b)
+{
+	const char *ca = (const char *) a;
+	const char *cb = (const char *) b;
+
+	return strcmp(ca, cb);
+}
+
+#define TC_SYS	"tcrunch"
+
+PyObject *PyFtrace_tc_event_system(PyObject *self)
+{
+	return PyUnicode_FromString(TC_SYS);
+}
+
+static int unregister_kprobe(const char *event)
+{
+	return tracefs_kprobe_clear_probe(TC_SYS, event, true);
+}
+
+void kprobe_free(void *kp)
+{
+	char *event = kp;
+
+	if (unregister_kprobe(event) < 0)
+		fprintf(stderr, "\ntfs_error: Failed to unregister kprobe \'%s\'.\n",
+		        event);
+
+	free(kp);
+}
+
+static void destroy_all_kprobes(void)
+{
+	tdestroy(kprobe_root, kprobe_free);
+	kprobe_root = NULL;
+}
+
+bool store_new_kprobe(const char *event)
+{
+	char *ptr = strdup(event);
+	char **val;
+
+	if (!ptr) {
+		MEM_ERROR;
+		return false;
+	}
+
+	val = tsearch(ptr, &kprobe_root, kprobe_compare);
+	if (!val || strcmp(*val, ptr) != 0) {
+		PyErr_Format(TFS_ERROR, "Failed to store new kprobe \'%s\'.",
+			     event);
+		return false;
+	}
+
+	return true;
+}
+
+PyObject *PyFtrace_register_kprobe(PyObject *self, PyObject *args,
+						   PyObject *kwargs)
+{
+	static char *kwlist[] = {"event", "function", "probe", NULL};
+	const char *event, *function, *probe;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "sss",
+					 kwlist,
+					 &event,
+					 &function,
+					 &probe)) {
+		return NULL;
+	}
+
+	if (tracefs_kprobe_raw(TC_SYS, event, function, probe) < 0) {
+		PyErr_Format(TFS_ERROR, "Failed to register kprobe \'%s\'.",
+			     event);
+		return NULL;
+	}
+
+	if (!store_new_kprobe(event))
+		return NULL;
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_register_kretprobe(PyObject *self, PyObject *args,
+						      PyObject *kwargs)
+{
+	static char *kwlist[] = {"event", "function", "probe", NULL};
+	const char *event, *function, *probe = "$retval";
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "ss|s",
+					 kwlist,
+					 &event,
+					 &function,
+					 &probe)) {
+		return NULL;
+	}
+
+	if (tracefs_kretprobe_raw(TC_SYS, event, function, probe) < 0) {
+		PyErr_Format(TFS_ERROR, "Failed to register kretprobe \'%s\'.",
+			     event);
+		return NULL;
+	}
+
+	if (!store_new_kprobe(event))
+		return NULL;
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_unregister_kprobe(PyObject *self, PyObject *args,
+						     PyObject *kwargs)
+{
+	static char *kwlist[] = {"event", "force",  NULL};
+	const char *event;
+	int force = false;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s|p",
+					 kwlist,
+					 &event,
+					 &force)) {
+		return NULL;
+	}
+
+	if (is_all(event)) {
+		if (force) {
+			/* Clear all register kprobes. */
+			if (tracefs_kprobe_clear_all(force) < 0)
+				goto fail;
+		} else {
+			/*
+			 * Clear only the kprobes registered by
+			 * trace-cruncher.
+			 */
+			destroy_all_kprobes();
+		}
+	} else {
+		tdelete(event, &kprobe_root, kprobe_compare);
+		if (unregister_kprobe(event) < 0)
+			goto fail;
+	}
+
+	Py_RETURN_NONE;
+
+ fail:
+	PyErr_Format(TFS_ERROR, "Failed to unregister kprobe \'%s\'.", event);
+	return NULL;
+}
+
+PyObject *PyFtrace_registered_kprobe_names(PyObject *self)
+{
+	char **list = tracefs_get_kprobes(TRACEFS_ALL_KPROBES);
+	return tfs_list2py_list(list);
+}
+
+PyObject *PyFtrace_registered_kprobes(PyObject *self)
+{
+	const char *file = "kprobe_events";
+	PyObject *list = PyList_New(0);
+	char *probes, *token;
+	int size;
+
+	size = read_from_file(NULL, file, &probes);
+	if (size < 0)
+		return NULL;
+
+	if (size == 0 || !probes)
+		return list;
+
+	token = strtok(probes, "\n");
+	while (token != NULL) {
+		PyList_Append(list, PyUnicode_FromString(token));
+		token = strtok(NULL, "\n");
+	}
+
+	return list;
+}
+
+PyObject *PyFtrace_set_kprobe_filter(PyObject *self, PyObject *args,
+						     PyObject *kwargs)
+{
+	const char *instance_name = NO_ARG, *event, *filter;
+	struct tracefs_instance *instance;
+	char path[PATH_MAX];
+
+	static char *kwlist[] = {"event", "filter", "instance", NULL};
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "ss|s",
+					 kwlist,
+					 &event,
+					 &filter,
+					 &instance_name)) {
+		return NULL;
+	}
+
+	if (!get_optional_instance(instance_name, &instance))
+		return NULL;
+
+	sprintf(path, "events/%s/%s/filter", TC_SYS, event);
+	if (!write_to_file_and_check(instance, path, filter)) {
+		PyErr_SetString(TFS_ERROR, "Failed to set kprobe filter.");
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_clear_kprobe_filter(PyObject *self, PyObject *args,
+						       PyObject *kwargs)
+{
+	const char *instance_name = NO_ARG, *event;
+	struct tracefs_instance *instance;
+	char path[PATH_MAX];
+
+	static char *kwlist[] = {"event", "instance", NULL};
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s|s",
+					 kwlist,
+					 &event,
+					 &instance_name)) {
+		return NULL;
+	}
+
+	if (!get_optional_instance(instance_name, &instance))
+		return NULL;
+
+	sprintf(path, "events/%s/%s/filter", TC_SYS, event);
+	if (!write_to_file(instance, path, OFF)) {
+		PyErr_SetString(TFS_ERROR, "Failed to clear kprobe filter.");
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
+static bool enable_kprobe(PyObject *self, PyObject *args, PyObject *kwargs,
+			  bool enable)
+{
+	static char *kwlist[] = {"event", "instance", NULL};
+	struct tracefs_instance *instance;
+	const char *instance_name, *event;
+
+	instance_name = event = NO_ARG;
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s|s",
+					 kwlist,
+					 &event,
+					 &instance_name)) {
+		return false;
+	}
+
+	if (!get_optional_instance(instance_name, &instance))
+		return false;
+
+	return event_enable_disable(instance, TC_SYS, event, enable);
+}
+
+PyObject *PyFtrace_enable_kprobe(PyObject *self, PyObject *args,
+						 PyObject *kwargs)
+{
+	if (!enable_kprobe(self, args, kwargs, true))
+		return NULL;
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_disable_kprobe(PyObject *self, PyObject *args,
+						  PyObject *kwargs)
+{
+	if (!enable_kprobe(self, args, kwargs, false))
+		return NULL;
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_kprobe_is_enabled(PyObject *self, PyObject *args,
+						     PyObject *kwargs)
+{
+	static char *kwlist[] = {"event", "instance", NULL};
+	struct tracefs_instance *instance;
+	const char *instance_name, *event;
+
+	instance_name = event = NO_ARG;
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s|s",
+					 kwlist,
+					 &event,
+					 &instance_name)) {
+		return NULL;
+	}
+
+	if (!get_optional_instance(instance_name, &instance))
+		return NULL;
+
+	return event_is_enabled(instance, TC_SYS, event);
+}
+
 static bool set_fork_options(struct tracefs_instance *instance, bool enable)
 {
 	if (enable) {
@@ -1865,5 +2194,6 @@ PyObject *PyFtrace_hook2pid(PyObject *self, PyObject *args, PyObject *kwargs)
 
 void PyFtrace_at_exit(void)
 {
+	destroy_all_kprobes();
 	destroy_all_instances();
 }
src/ftracepy-utils.h +30 -0
index 3699aaa..d826427 100644
@@ -125,6 +125,36 @@ PyObject *PyFtrace_supported_options(PyObject *self, PyObject *args,
 PyObject *PyFtrace_enabled_options(PyObject *self, PyObject *args,
 						   PyObject *kwargs);
 
+PyObject *PyFtrace_tc_event_system(PyObject *self);
+
+PyObject *PyFtrace_register_kprobe(PyObject *self, PyObject *args,
+						   PyObject *kwargs);
+
+PyObject *PyFtrace_register_kretprobe(PyObject *self, PyObject *args,
+						      PyObject *kwargs);
+
+PyObject *PyFtrace_unregister_kprobe(PyObject *self, PyObject *args,
+						     PyObject *kwargs);
+
+PyObject *PyFtrace_registered_kprobes(PyObject *self);
+
+PyObject *PyFtrace_registered_kprobe_names(PyObject *self);
+
+PyObject *PyFtrace_set_kprobe_filter(PyObject *self, PyObject *args,
+						     PyObject *kwargs);
+
+PyObject *PyFtrace_clear_kprobe_filter(PyObject *self, PyObject *args,
+						       PyObject *kwargs);
+
+PyObject *PyFtrace_enable_kprobe(PyObject *self, PyObject *args,
+						 PyObject *kwargs);
+
+PyObject *PyFtrace_disable_kprobe(PyObject *self, PyObject *args,
+						  PyObject *kwargs);
+
+PyObject *PyFtrace_kprobe_is_enabled(PyObject *self, PyObject *args,
+						     PyObject *kwargs);
+
 PyObject *PyFtrace_trace_process(PyObject *self, PyObject *args,
 						 PyObject *kwargs);
 
src/ftracepy.c +55 -0

61 lines not on this page · raw message

tests/1_unit/test_01_ftracepy_unit.py +57 -0

63 lines not on this page · raw message

Signature · 1 line
2.27.0
#2 lore raw

PATCH v2 2/3 trace-cruncher: Add events to utils

In reply to Yordan Karadzhov (VMware)

Define a hierarchy of Python classes, to be used for easy manipulation of Staic events and Kprobes.

---
 tracecruncher/ft_utils.py | 182 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
tracecruncher/ft_utils.py +182 -0
index eae161c..5b4f2f2 100644
@@ -6,6 +6,7 @@ Copyright 2019 VMware Inc, Yordan Karadzhov (VMware) <[email protected]>
 
 import sys
 import time
+import ctypes
 
 from . import ftracepy as ft
 
@@ -17,3 +18,184 @@ def find_event_id(system, event):
     tep.init_local(dir=ft.dir(), systems=[system]);
 
     return tep.get_event(system=system, name=event).id()
+
+
+class event:
+    def __init__(self, system, name, static=True):
+        """ Constructor.
+        """
+        self.system = system
+        self.name = name
+        self.instance_list = []
+        if static:
+            self.evt_id = find_event_id(system, name)
+        else:
+            self.evt_id = -1
+
+    def id(self):
+        """ Retrieve the unique ID of the kprobe event.
+        """
+        return int(self.evt_id)
+
+    def enable(self, instance=None):
+        """ Enable this event.
+        """
+        if instance is None:
+            ft.enable_event(system=self.system, event=self.name)
+            self.instance_list.append('top')
+        else:
+            ft.enable_event(instance=instance, system=self.system, event=self.name)
+            self.instance_list.append(instance)
+
+        self.instance_list = list(set(self.instance_list))
+
+    def disable(self, instance=None):
+        """ Disable this event.
+        """
+        if instance is None:
+            ft.disable_event(system=self.system, event=self.name)
+            self.instance_list.remove('top')
+        else:
+            ft.disable_event(instance=instance,system=self.system, event=self.name)
+            self.instance_list.remove(instance)
+
+    def set_filter(self, filter, instance=None):
+        """ Define a filter for this event.
+        """
+        if instance is None:
+            ft.set_event_filter(system=self.system,
+                                event=self.name,
+                                filter=filter)
+        else:
+            ft.set_event_filter(instance=instance,
+                                system=self.system,
+                                event=self.name,
+                                filter=filter)
+
+    def clear_filter(self, instance=None):
+        """ Define the filter for this event.
+        """
+        if instance is None:
+            ft.clear_event_filter(system=self.system,
+                                  event=self.name)
+        else:
+            ft.clear_event_filter(instance=instance,
+                                  system=self.system,
+                                  event=self.name)
+
+
+class kprobe_base(event):
+    def __init__(self, name, func=''):
+        """ Constructor.
+        """
+        super().__init__(system=ft.tc_event_system(), name=name, static=False)
+        self.func = func
+
+    def set_function(self, name):
+        """ Set the name of the function to be traced.
+        """
+        self.func = name
+
+    def unregister(self):
+        """ Unregister this probe from Ftrace.
+        """
+        inst_list = self.instance_list.copy()
+        for instance in inst_list:
+            self.disable(instance)
+
+        ft.unregister_kprobe(event=self.name);
+
+
+class kprobe(kprobe_base):
+    def __init__(self, name, func=''):
+        """ Constructor.
+        """
+        super().__init__(name, func)
+        self.fields = {}
+
+    def add_raw_field(self, name, probe):
+        """ Add a raw definition of a data field to this probe.
+        """
+        self.fields[str(name)] = str(probe)
+
+    def add_arg(self, name, param_id, param_type):
+        """ Add a function parameter data field to this probe.
+        """
+        probe = '$arg{0}:{1}'.format(param_id, param_type)
+        self.add_raw_field(name, probe)
+
+    def add_ptr_arg(self, name, param_id, param_type, offset=0):
+        """ Add a pointer function parameter data field to this probe.
+        """
+        probe = '+{0}($arg{1}):{2}'.format(offset, param_id, param_type)
+        self.add_raw_field(name, probe)
+
+    def add_array_arg(self, name, param_id, param_type, offset=0, size=-1):
+        """ Add a array parameter data field to this probe.
+        """
+        if size < 0:
+            size = 10
+
+        ptr_size = ctypes.sizeof(ctypes.c_voidp)
+        for i in range(size):
+            field_name = name + str(i)
+            probe = '+{0}(+{1}'.format(offset, i * ptr_size)
+            probe += '($arg{0})):{1}'.format(param_id, param_type)
+            self.add_raw_field(field_name, probe)
+
+    def add_string_arg(self, name, param_id, offset=0, usr_space=False):
+        """ Add a pointer function parameter data field to this probe.
+        """
+        p_type = 'ustring' if usr_space else 'string'
+        self.add_ptr_arg(name=name,
+                         param_id=param_id,
+                         param_type=p_type,
+                         offset=offset)
+
+    def add_string_array_arg(self, name, param_id, offset=0, usr_space=False, size=-1):
+        """ Add a string array parameter data field to this probe.
+        """
+        p_type = 'ustring' if usr_space else 'string'
+        self.add_array_arg(name=name,
+                           param_id=param_id,
+                           param_type=p_type,
+                           offset=offset,
+                           size=size)
+
+    def register(self):
+        """ Register this probe to Ftrace.
+        """
+        probe = ' '.join('{!s}={!s}'.format(key,val) for (key, val) in self.fields.items())
+
+        ft.register_kprobe(event=self.name, function=self.func, probe=probe);
+        self.evt_id = find_event_id(system=ft.tc_event_system(), event=self.name)
+
+
+def parse_record_array_field(event, record, field, size=-1):
+    """ Register this probe to Ftrace.
+    """
+    if size < 0:
+        size = 10
+
+    arr = []
+    for i in range(size):
+        field_name = field + str(i)
+        val = event.parse_record_field(record=record, field=field_name)
+        if (val == '(nil)'):
+            break
+        arr.append(val)
+
+    return arr
+
+
+class kretval_probe(kprobe_base):
+    def __init__(self, name, func=''):
+        """ Constructor.
+        """
+        super().__init__(name, func)
+
+    def register(self):
+        """ Register this probe to Ftrace.
+        """
+        ft.register_kprobe(event=self.name, function=self.func);
+        self.evt_id = find_event_id(system=ft.tc_event_system(), event=self.name)
Signature · 1 line
2.27.0
#3 lore raw

PATCH v2 3/3 trace-cruncher: Add Kprobe example

In reply to Yordan Karadzhov (VMware)

This is a very basic example that aims to demonstrate the usage of Kprobes in trace-cruncher.

---
 examples/kprobe_open.py | 44 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100755 examples/kprobe_open.py
examples/kprobe_open.py new file +44 -0
new file mode 100755
index 0000000..f8d4d8b
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+"""
+SPDX-License-Identifier: CC-BY-4.0
+
+Copyright (C) 2021 VMware Inc, Yordan Karadzhov (VMware) <[email protected]>
+"""
+
+import sys
+
+import tracecruncher.ftracepy as ft
+import tracecruncher.ft_utils as tc
+
+open_probe = tc.kprobe(name='open', func='do_sys_openat2')
+
+open_probe.add_string_arg(name='file', param_id=2)
+
+open_probe.add_ptr_arg(name='flags',
+                       param_id=3,
+                       param_type='x64')
+
+open_probe.add_ptr_arg(name='mode',
+                       param_id=3,
+                       param_type='x64',
+                       offset=8)
+
+open_probe.register()
+
+def callback(event, record):
+    if event.id() == open_probe.id():
+        file_name = event.parse_record_field(record=record, field='file')
+        flags = event.parse_record_field(record, 'flags')
+        mode = event.parse_record_field(record, 'mode')
+        print('file: {0}  (flags: {1}; mode: {2})'.format(file_name, hex(flags), hex(mode)))
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print('Usage: ', sys.argv[0], ' [PROCESS]')
+        sys.exit(1)
+
+    inst = ft.create_instance(tracing_on=False)
+    open_probe.enable(instance=inst)
+    ft.trace_process(instance=inst, argv=sys.argv[1:])
Signature · 1 line
2.27.0