trace-cruncher: Add Kprobes
cover trace-cruncher: Add Kprobes
This adds to trace-cruncher a basic support for Kprobes.
Yordan Karadzhov (VMware) (3):
trace-cruncher: Add support for Kprobes
trace-cruncher: Add events to utils
trace-cruncher: Add Kprobe example
examples/kprobe_open.py | 44 ++++ src/ftracepy-utils.c | 366 ++++++++++++++++++++++++-- src/ftracepy-utils.h | 30 +++ src/ftracepy.c | 55 ++++ tests/1_unit/test_01_ftracepy_unit.py | 56 ++++ tracecruncher/ft_utils.py | 182 +++++++++++++ 6 files changed, 715 insertions(+), 18 deletions(-) create mode 100755 examples/kprobe_open.py
Signature · 1 line
2.27.0
PATCH 1/3 trace-cruncher: Add support for Kprobes
Kprobes is a built-in debugging mechanism for the Linux kernel, which can be used extract monitoring data from a running production system. Here we add to trace-cruncher a basic support for using Kprobes.
--- src/ftracepy-utils.c | 366 ++++++++++++++++++++++++-- src/ftracepy-utils.h | 30 +++ src/ftracepy.c | 55 ++++ tests/1_unit/test_01_ftracepy_unit.py | 56 ++++ 4 files changed, 489 insertions(+), 18 deletions(-)
src/ftracepy-utils.c +348 -18
@@ -100,6 +100,7 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args, PyObject *kwargs) {struct tep_format_field *field; +int field_offset, field_size;const char *field_name;PyTepRecord *record;
@@ -124,11 +125,24 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args,return NULL ; } -if (!field->size) +if (field->flags &TEP_FIELD_IS_DYNAMIC ) { +unsigned long long val; + + val =tep_read_number (self->ptrObj->tep, + record->ptrObj->data + field->offset, + field->size); + field_offset = val &0xffff ; + field_size = val >>16 ; + }else { + field_offset = field->offset; + field_size = field->size; + } + +if (!field_size)return PyUnicode_FromString ("(nil)" );if (field->flags &TEP_FIELD_IS_STRING ) { -char *val_str = record->ptrObj->data + field->offset; +char *val_str = record->ptrObj->data + field_offset;return PyUnicode_FromString (val_str); }else if (is_number (field)) {unsigned long long val;
@@ -136,7 +150,7 @@ PyObject *PyTepEvent_parse_record_field(PyTepEvent* self, PyObject *args,tep_read_number_field (field, record->ptrObj->data, &val);return PyLong_FromLong (val); }else if (field->flags &TEP_FIELD_IS_POINTER ) { -void *val = record->ptrObj->data + field->offset; +void *val = record->ptrObj->data + field_offset;char ptr_string[11 ];sprintf (ptr_string,"%p" , val);
@@ -1058,14 +1072,33 @@ PyObject *PyFtrace_disable_events(PyObject *self, PyObject *args, Py_RETURN_NONE; } +static PyObject *event_is_enabled (struct tracefs_instance *instance, +const char *system,const char *event) +{ +char *file, *val; +PyObject *ret; + +if (!get_event_enable_file (instance, system, event, &file)) +return NULL ; + +if (read_from_file (instance, file, &val) <=0 ) +return NULL ; + +trim_new_line (val); + ret =PyUnicode_FromString (val); + +free (file); +free (val); + +return ret; +} + PyObject *PyFtrace_event_is_enabled(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"instance", "system", "event", NULL}; const char *instance_name, *system, *event; struct tracefs_instance *instance; - char *file, *val; - PyObject *ret; instance_name = system = event = NO_ARG; if (!PyArg_ParseTupleAndKeywords(args,
@@ -1081,19 +1114,7 @@ PyObject *PyFtrace_event_is_enabled(PyObject *self, PyObject *args,if (!get_optional_instance (instance_name, &instance))return false; -if (!get_event_enable_file (instance, system, event, &file)) -return NULL ; - -if (read_from_file (instance, file, &val) <=0 ) -return NULL ; - -trim_new_line (val); - ret =PyUnicode_FromString (val); - -free (file); -free (val); - -return ret; +return event_is_enabled (instance, system, event); } PyObject *PyFtrace_set_event_filter(PyObject *self, PyObject *args,
@@ -1472,6 +1493,314 @@ PyObject *PyFtrace_supported_options(PyObject *self, PyObject *args,return get_option_list (instance, false); } +static void *kprobe_root =NULL ; + +static int kprobe_compare (const void *a,const void *b) +{ +const char *ca = (const char *) a; +const char *cb = (const char *) b; + +return strcmp (ca, cb); +} + +#define TC_SYS "tcrunch" + +PyObject *PyFtrace_tc_event_system (PyObject *self) +{ +return PyUnicode_FromString (TC_SYS ); +} + +static int unregister_kprobe (const char *event) +{ +return tracefs_kprobe_clear_probe (TC_SYS , event, true); +} + +void kprobe_free (void *kp) +{ +char *event = kp; + +if (unregister_kprobe (event) <0 ) +fprintf (stderr,"\ntfs_error: Failed to unregister kprobe \'%s\'.\n" , + event); + +free (kp); +} + +static void destroy_all_kprobes (void ) +{ +tdestroy (kprobe_root, kprobe_free); + kprobe_root =NULL ; +} + +bool store_new_kprobe (const char *event) +{ +char *ptr =strdup (event); +char **val; + +if (!ptr) { +MEM_ERROR ; +return false; + } + + val =tsearch (ptr, &kprobe_root, kprobe_compare); +if (!val ||strcmp (*val, ptr) !=0 ) { +PyErr_Format (TFS_ERROR ,"Failed to store new kprobe \'%s\'." , + event); +return false; + } + +return true; +} + +PyObject *PyFtrace_register_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +static char *kwlist[] = {"event" ,"function" ,"probe" ,NULL }; +const char *event, *function, *probe; + +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"sss" , + kwlist, + &event, + &function, + &probe)) { +return NULL ; + } + +if (tracefs_kprobe_raw (TC_SYS , event, function, probe) <0 ) { +PyErr_Format (TFS_ERROR ,"Failed to register kprobe \'%s\'." , + event); +return NULL ; + } + +if (!store_new_kprobe (event)) +return NULL ; + + Py_RETURN_NONE; +} + +PyObject *PyFtrace_register_kretprobe (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +static char *kwlist[] = {"event" ,"function" ,"probe" ,NULL }; +const char *event, *function, *probe ="$retval" ; + +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"ss|s" , + kwlist, + &event, + &function, + &probe)) { +return NULL ; + } + +if (tracefs_kretprobe_raw (TC_SYS , event, function, probe) <0 ) { +PyErr_Format (TFS_ERROR ,"Failed to register kretprobe \'%s\'." , + event); +return NULL ; + } + +if (!store_new_kprobe (event)) +return NULL ; + + Py_RETURN_NONE; +} + +PyObject *PyFtrace_unregister_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +static char *kwlist[] = {"event" ,"force" ,NULL }; +const char *event; +int force = false; + +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"s|p" , + kwlist, + &event, + &force)) { +return NULL ; + } + +if (is_all (event)) { +if (force) { +/* Clear all register kprobes. */ +if (tracefs_kprobe_clear_all (force) <0 ) + goto fail; + }else { +/* +* Clear only the kprobes registered by +* trace-cruncher. +*/ +destroy_all_kprobes (); + } + }else { +tdelete (event, &kprobe_root, kprobe_compare); +if (unregister_kprobe (event) <0 ) + goto fail; + } + + Py_RETURN_NONE; + + fail: +PyErr_Format (TFS_ERROR ,"Failed to unregister kprobe \'%s\'." , event); +return NULL ; +} + +PyObject *PyFtrace_registered_kprobe_names (PyObject *self) +{ +char **list =tracefs_get_kprobes (TRACEFS_ALL_KPROBES ); +return tfs_list2py_list (list); +} + +PyObject *PyFtrace_registered_kprobes (PyObject *self) +{ +const char *file ="kprobe_events" ; +PyObject *list =PyList_New (0 ); +char *probes, *token; +int size; + + size =read_from_file (NULL , file, &probes); +if (size <0 ) +return NULL ; + +if (size ==0 || !probes) +return list; + + token =strtok (probes,"\n" ); +while (token !=NULL ) { +PyList_Append (list,PyUnicode_FromString (token)); + token =strtok (NULL ,"\n" ); + } + +return list; +} + +PyObject *PyFtrace_set_kprobe_filter (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +const char *instance_name =NO_ARG , *event, *filter; +struct tracefs_instance *instance; +char path[PATH_MAX ]; + +static char *kwlist[] = {"event" ,"filter" ,"instance" ,NULL }; +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"ss|s" , + kwlist, + &event, + &filter, + &instance_name)) { +return NULL ; + } + +if (!get_optional_instance (instance_name, &instance)) +return NULL ; + +sprintf (path,"events/%s/%s/filter" ,TC_SYS , event); +if (!write_to_file_and_check (instance, path, filter)) { +PyErr_SetString (TFS_ERROR ,"Failed to set kprobe filter." ); +return NULL ; + } + + Py_RETURN_NONE; +} + +PyObject *PyFtrace_clear_kprobe_filter (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +const char *instance_name =NO_ARG , *event; +struct tracefs_instance *instance; +char path[PATH_MAX ]; + +static char *kwlist[] = {"event" ,"instance" ,NULL }; +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"s|s" , + kwlist, + &event, + &instance_name)) { +return NULL ; + } + +if (!get_optional_instance (instance_name, &instance)) +return NULL ; + +sprintf (path,"events/%s/%s/filter" ,TC_SYS , event); +if (!write_to_file (instance, path,OFF )) { +PyErr_SetString (TFS_ERROR ,"Failed to clear kprobe filter." ); +return NULL ; + } + + Py_RETURN_NONE; +} + +static bool enable_kprobe (PyObject *self,PyObject *args,PyObject *kwargs, +bool enable) +{ +static char *kwlist[] = {"event" ,"instance" ,NULL }; +struct tracefs_instance *instance; +const char *instance_name, *event; + + instance_name = event =NO_ARG ; +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"s|s" , + kwlist, + &event, + &instance_name)) { +return false; + } + +if (!get_optional_instance (instance_name, &instance)) +return false; + +return event_enable_disable (instance,TC_SYS , event, enable); +} + +PyObject *PyFtrace_enable_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +if (!enable_kprobe (self, args, kwargs, true)) +return NULL ; + + Py_RETURN_NONE; +} + +PyObject *PyFtrace_disable_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +if (!enable_kprobe (self, args, kwargs, false)) +return NULL ; + + Py_RETURN_NONE; +} + +PyObject *PyFtrace_kprobe_is_enabled (PyObject *self,PyObject *args, +PyObject *kwargs) +{ +static char *kwlist[] = {"event" ,"instance" ,NULL }; +struct tracefs_instance *instance; +const char *instance_name, *event; + + instance_name = event =NO_ARG ; +if (!PyArg_ParseTupleAndKeywords (args, + kwargs, +"s|s" , + kwlist, + &event, + &instance_name)) { +return NULL ; + } + +if (!get_optional_instance (instance_name, &instance)) +return NULL ; + +return event_is_enabled (instance,TC_SYS , event); +} +static bool set_fork_options (struct tracefs_instance *instance,bool enable) {if (enable) {
@@ -1865,5 +2194,6 @@ PyObject *PyFtrace_hook2pid(PyObject *self, PyObject *args, PyObject *kwargs)void PyFtrace_at_exit (void ) { +destroy_all_kprobes ();destroy_all_instances (); }
src/ftracepy-utils.h +30 -0
@@ -125,6 +125,36 @@ PyObject *PyFtrace_supported_options(PyObject *self, PyObject *args,PyObject *PyFtrace_enabled_options (PyObject *self,PyObject *args,PyObject *kwargs); +PyObject *PyFtrace_tc_event_system (PyObject *self); + +PyObject *PyFtrace_register_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_register_kretprobe (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_unregister_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_registered_kprobes (PyObject *self); + +PyObject *PyFtrace_registered_kprobe_names (PyObject *self); + +PyObject *PyFtrace_set_kprobe_filter (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_clear_kprobe_filter (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_enable_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_disable_kprobe (PyObject *self,PyObject *args, +PyObject *kwargs); + +PyObject *PyFtrace_kprobe_is_enabled (PyObject *self,PyObject *args, +PyObject *kwargs); +PyObject *PyFtrace_trace_process (PyObject *self,PyObject *args,PyObject *kwargs);
src/ftracepy.c +55 -0
61 lines not on this page · raw message
tests/1_unit/test_01_ftracepy_unit.py +56 -0
62 lines not on this page · raw message
Signature · 1 line
2.27.0
PATCH 2/3 trace-cruncher: Add events to utils
In reply to Yordan Karadzhov (VMware)Define a hierarchy of Python classes, to be used for easy manipulation of Staic events and Kprobes.
--- tracecruncher/ft_utils.py | 182 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+)
tracecruncher/ft_utils.py +182 -0
@@ -6,6 +6,7 @@ Copyright 2019 VMware Inc, Yordan Karadzhov (VMware) <[email protected]> import sys import time +import ctypes from . import ftracepy as ft
@@ -17,3 +18,184 @@ def find_event_id(system, event): tep.init_local(dir=ft.dir(), systems=[system]); return tep.get_event(system=system, name=event).id() + + +class event: + def __init__(self, system, name, static=True): + """ Constructor. + """ + self.system = system + self.name = name + self.instance_list = [] + if static: + self.evt_id = find_event_id(system, name) + else: + self.evt_id = -1 + + def id(self): + """ Retrieve the unique ID of the kprobe event. + """ + return int(self.evt_id) + + def enable(self, instance=None): + """ Enable this event. + """ + if instance is None: + ft.enable_event(system=self.system, event=self.name) + self.instance_list.append('top') + else: + ft.enable_event(instance=instance, system=self.system, event=self.name) + self.instance_list.append(instance) + + self.instance_list = list(set(self.instance_list)) + + def disable(self, instance=None): + """ Disable this event. + """ + if instance is None: + ft.disable_event(system=self.system, event=self.name) + self.instance_list.remove('top') + else: + ft.disable_event(instance=instance,system=self.system, event=self.name) + self.instance_list.remove(instance) + + def set_filter(self, filter, instance=None): + """ Define a filter for this event. + """ + if instance is None: + ft.set_event_filter(system=self.system, + event=self.name, + filter=filter) + else: + ft.set_event_filter(instance=instance, + system=self.system, + event=self.name, + filter=filter) + + def clear_filter(self, instance=None): + """ Define the filter for this event. + """ + if instance is None: + ft.clear_event_filter(system=self.system, + event=self.name) + else: + ft.clear_event_filter(instance=instance, + system=self.system, + event=self.name) + + +class kprobe(event): + def __init__(self, name, func=''): + """ Constructor. + """ + super().__init__(system=ft.tc_event_system(), name=name, static=False) + self.func = func + + def set_function(self, name): + """ Set the name of the function to be traced. + """ + self.func = name + + def unregister(self): + """ Unregister this probe from Ftrace. + """ + inst_list = self.instance_list.copy() + for instance in inst_list: + self.disable(instance) + + ft.unregister_kprobe(event=self.name); + + +class kprobe(kprobe): + def __init__(self, name, func=''): + """ Constructor. + """ + super().__init__(name, func) + self.fields = {} + + def add_raw_field(self, name, probe): + """ Add a raw definition of a data field to this probe. + """ + self.fields[str(name)] = str(probe) + + def add_arg(self, name, param_id, param_type): + """ Add a function parameter data field to this probe. + """ + probe = '$arg{0}:{1}'.format(param_id, param_type) + self.add_raw_field(name, probe) + + def add_ptr_arg(self, name, param_id, param_type, offset=0): + """ Add a pointer function parameter data field to this probe. + """ + probe = '+{0}($arg{1}):{2}'.format(offset, param_id, param_type) + self.add_raw_field(name, probe) + + def add_array_arg(self, name, param_id, param_type, offset=0, size=-1): + """ Add a array parameter data field to this probe. + """ + if size < 0: + size = 10 + + ptr_size = ctypes.sizeof(ctypes.c_voidp) + for i in range(size): + field_name = name + str(i) + probe = '+{0}(+{1}'.format(offset, i * ptr_size) + probe += '($arg{0})):{1}'.format(param_id, param_type) + self.add_raw_field(field_name, probe) + + def add_string_arg(self, name, param_id, offset=0, usr_space=False): + """ Add a pointer function parameter data field to this probe. + """ + p_type = 'ustring' if usr_space else 'string' + self.add_ptr_arg(name=name, + param_id=param_id, + param_type=p_type, + offset=offset) + + def add_string_array_arg(self, name, param_id, offset=0, usr_space=False, size=-1): + """ Add a string array parameter data field to this probe. + """ + p_type = 'ustring' if usr_space else 'string' + self.add_array_arg(name=name, + param_id=param_id, + param_type=p_type, + offset=offset, + size=size) + + def register(self): + """ Register this probe to Ftrace. + """ + probe = ' '.join('{!s}={!s}'.format(key,val) for (key, val) in self.fields.items()) + + ft.register_kprobe(event=self.name, function=self.func, probe=probe); + self.evt_id = find_event_id(system=ft.tc_event_system(), event=self.name) + + +def parse_record_array_field(event, record, field, size=-1): + """ Register this probe to Ftrace. + """ + if size < 0: + size = 10 + + arr = [] + for i in range(size): + field_name = field + str(i) + val = event.parse_record_field(record=record, field=field_name) + if (val == '(nil)'): + break + arr.append(val) + + return arr + + +class kretval_probe(kprobe): + def __init__(self, name, func=''): + """ Constructor. + """ + super().__init__(name, func) + + def register(self): + """ Register this probe to Ftrace. + """ + ft.register_kprobe(event=self.name, function=self.func); + self.evt_id = find_event_id(system=ft.tc_event_system(), event=self.name)
Signature · 1 line
2.27.0
PATCH 3/3 trace-cruncher: Add Kprobe example
In reply to Yordan Karadzhov (VMware)This is a very basic example that aims to demonstrate the usage of Kprobes in trace-cruncher.
--- examples/kprobe_open.py | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 examples/kprobe_open.py
examples/kprobe_open.py new file +44 -0
@@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" +SPDX-License-Identifier: CC-BY-4.0 + +Copyright (C) 2021 VMware Inc, Yordan Karadzhov (VMware) <[email protected]> +""" + +import sys + +import tracecruncher.ftracepy as ft +import tracecruncher.ft_utils as tc + +open_probe = tc.kprobe(name='open', func='do_sys_openat2') + +open_probe.add_string_arg(name='file', param_id=2) + +open_probe.add_ptr_arg(name='flags', + param_id=3, + param_type='x64') + +open_probe.add_ptr_arg(name='mode', + param_id=3, + param_type='x64', + offset=8) + +open_probe.register() + +def callback(event, record): + if event.id() == open_probe.id(): + file_name = event.parse_record_field(record=record, field='file') + flags = event.parse_record_field(record, 'flags') + mode = event.parse_record_field(record, 'mode') + print('file: {0} (flags: {1}; mode: {2})'.format(file_name, hex(flags), hex(mode))) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print('Usage: ', sys.argv[0], ' [PROCESS]') + sys.exit(1) + + inst = ft.create_instance(tracing_on=False) + open_probe.enable(instance=inst) + ft.trace_process(instance=inst, argv=sys.argv[1:])
Signature · 1 line
2.27.0