perf: Reorganize scripting support
Subsystems (4)
Ian Rogers
reviewerPATCH v21 9/21 perf python: Add wrapper for perf_data file abstraction
In reply to Ian RogersThe perf_data struct is needed for session support.
--- tools/perf/util/python.c | 123 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-)
tools/perf/util/python.c +122 -1
@@ -6,6 +6,7 @@#include <linux/err.h> #include <poll.h> +#include <unistd.h> #include <internal/lib.h> #include <perf/cpumap.h>
@@ -14,6 +15,7 @@#include "callchain.h" #include "counts.h" +#include "data.h" #include "event.h" #include "evlist.h" #include "evsel.h"
@@ -2462,6 +2464,121 @@ static PyObject *pyrf__metrics(PyObject *self, PyObject *args)return list; } +struct pyrf_data { +PyObject_HEAD + + struct perf_data data; +}; + +static int pyrf_data__init (struct pyrf_data *pdata,PyObject *args,PyObject *kwargs) +{ +static char *kwlist[] = {"path" ,"fd" ,NULL }; +char *path =NULL ; +int fd =-1 ; + +if (!PyArg_ParseTupleAndKeywords (args, kwargs,"|si" , kwlist, &path, &fd)) +return -1 ; + +if (pdata->data.open) +perf_data__close (&pdata->data); +free ((char *)pdata->data.path); + pdata->data.path =NULL ; + +if (fd !=-1 ) { +struct stat st; + +if (fstat (fd, &st) <0 || !S_ISFIFO (st.st_mode)) { +PyErr_SetString (PyExc_ValueError, +"fd argument is only supported for pipes" ); +return -1 ; + } +if (!path) + path ="-" ; +else if (strcmp (path,"-" ) !=0 ) { +PyErr_SetString (PyExc_ValueError, +"path must be '-' when fd is provided" ); +return -1 ; + } + fd =dup (fd); +if (fd <0 ) { +PyErr_SetFromErrno (PyExc_OSError); +return -1 ; + } + }else if (path &&strcmp (path,"-" ) ==0 ) { + fd =dup (0 ); +if (fd <0 ) { +PyErr_SetFromErrno (PyExc_OSError); +return -1 ; + } + } + +if (!path) + path ="perf.data" ; + + pdata->data.path =strdup (path); +if (!pdata->data.path) { +if (fd !=-1 ) +close (fd); +PyErr_NoMemory (); +return -1 ; + } + + pdata->data.mode =PERF_DATA_MODE_READ ; + pdata->data.file.fd = fd; +if (perf_data__open (&pdata->data) <0 ) { +PyErr_Format (PyExc_IOError,"Failed to open perf data: %s" , + pdata->data.path ? pdata->data.path :"perf.data" ); +return -1 ; + } +return 0 ; +} + +static void pyrf_data__delete (struct pyrf_data *pdata) +{ +perf_data__close (&pdata->data); +free ((char *)pdata->data.path); +Py_TYPE (pdata)->tp_free ((PyObject *)pdata); +} + +static PyObject *pyrf_data__str (PyObject *self) +{ +const struct pyrf_data *pdata = (const struct pyrf_data *)self; + +if (!pdata->data.path) +return PyUnicode_FromString ("[uninitialized]" ); +return PyUnicode_FromString (pdata->data.path); +} + +static PyObject *pyrf_data__new (PyTypeObject *type,PyObject *args,PyObject *kwargs) +{ +struct pyrf_data *pdata; + + pdata = (struct pyrf_data *)PyType_GenericNew (type, args, kwargs); +if (pdata) +memset (&pdata->data,0 ,sizeof (pdata->data)); +return (PyObject *)pdata; +} + +static const char pyrf_data__doc[] =PyDoc_STR ("perf data file object." ); + +static PyTypeObject pyrf_data__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.data" , + .tp_basicsize =sizeof (struct pyrf_data ), + .tp_dealloc = (destructor )pyrf_data__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_data__doc, + .tp_init = (initproc )pyrf_data__init, + .tp_repr = pyrf_data__str, + .tp_str = pyrf_data__str, +}; + +static int pyrf_data__setup_types (void ) +{ + pyrf_data__type.tp_new = pyrf_data__new; +return PyType_Ready (&pyrf_data__type); +} +static PyMethodDef perf__methods[] = { { .ml_name ="metrics" ,
@@ -2524,7 +2641,8 @@ PyMODINIT_FUNC PyInit_perf(void)pyrf_cpu_map__setup_types () <0 ||pyrf_pmu_iterator__setup_types () <0 ||pyrf_pmu__setup_types () <0 || - pyrf_counts_values__setup_types() < 0) +pyrf_counts_values__setup_types () <0 || + pyrf_data__setup_types() < 0) return module;/* The page_size is placed in util object. */
@@ -2572,6 +2690,9 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_counts_values__type);PyModule_AddObject (module,"counts_values" , (PyObject *)&pyrf_counts_values__type); +Py_INCREF (&pyrf_data__type); +PyModule_AddObject (module,"data" , (PyObject *)&pyrf_data__type); + dict =PyModule_GetDict (module);if (dict ==NULL ) goto error;
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 10/21 perf python: Add python session abstraction wrapping perf's session
In reply to Ian RogersSessions are necessary to be able to use perf.data files within a tool. Add a wrapper python type that incorporates the tool. Allow a sample callback to be passed when creating the session. When process_events is run this callback will be called, if supplied, for sample events.
An example use looks like:
```
$ perf record -e cycles,instructions -a sleep 3
$ PYTHONPATH=..../perf/python python3
Python 3.13.7 (main, Aug 20 2025, 22:17:40) [GCC 14.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import perf
>>> count=0
... def handle_sample(x):
... global count
... if count < 3:
... print(dir(x))
... count = count + 1
... perf.session(perf.data("perf.data"),sample=handle_sample).process_events()
...
['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'sample_addr', 'sample_cpu', 'sample_id', 'sample_ip', 'sample_period', 'sample_pid', 'sample_stream_id', 'sample_tid', 'sample_time', 'type']
['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'sample_addr', 'sample_cpu', 'sample_id', 'sample_ip', 'sample_period', 'sample_pid', 'sample_stream_id', 'sample_tid', 'sample_time', 'type']
['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'sample_addr', 'sample_cpu', 'sample_id', 'sample_ip', 'sample_period', 'sample_pid', 'sample_stream_id', 'sample_tid', 'sample_time', 'type']
```Also, add the ability to get the thread associated with a session. For threads, allow the comm string to be retrieved. This can be useful for filtering threads. Connect up some of the standard event handling in psession->tool to better support queries of the machine. Also connect up the symbols.
--- tools/perf/util/python.c | 357 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 351 insertions(+), 6 deletions(-)
tools/perf/util/python.c +351 -6
@@ -14,8 +14,10 @@#include <structmember.h> #include "callchain.h" +#include "comm.h" #include "counts.h" #include "data.h" +#include "debug.h" #include "event.h" #include "evlist.h" #include "evsel.h"
@@ -25,8 +27,14 @@#include "pmus.h" #include "print_binary.h" #include "record.h" +#include "session.h" #include "strbuf.h" +#include "symbol.h" +#include "stat.h" +#include "header.h" +#include "thread.h" #include "thread_map.h" +#include "tool.h" #include "tp_pmu.h" #include "trace-event.h" #include "util/sample.h"
@@ -2579,6 +2587,332 @@ static int pyrf_data__setup_types(void)return PyType_Ready (&pyrf_data__type); } +struct pyrf_thread { +PyObject_HEAD + + struct thread *thread; +}; + +static void pyrf_thread__delete (struct pyrf_thread *pthread) +{ +thread__put (pthread->thread); +Py_TYPE (pthread)->tp_free ((PyObject *)pthread); +} + +static PyObject *pyrf_thread__comm (PyObject *obj) +{ +struct pyrf_thread *pthread = (void *)obj; +const char *str; + +CHECK_INITIALIZED (pthread->thread,"perf.thread" ); + + str =thread__comm_str (pthread->thread); + +if (!str) + Py_RETURN_NONE; + +return PyUnicode_FromString (str); +} + +static PyMethodDef pyrf_thread__methods[] = { + { + .ml_name ="comm" , + .ml_meth = (PyCFunction )pyrf_thread__comm, + .ml_flags =METH_NOARGS , + .ml_doc =PyDoc_STR ("Comm(and) associated with this thread." ) + }, + { .ml_name =NULL , } +}; + +static PyObject *pyrf_thread__get_pid (struct pyrf_thread *pthread,void *closure__maybe_unused ) +{ +return PyLong_FromLong (thread__pid (pthread->thread)); +} + +static PyObject *pyrf_thread__get_tid (struct pyrf_thread *pthread,void *closure__maybe_unused ) +{ +return PyLong_FromLong (thread__tid (pthread->thread)); +} + +static PyObject *pyrf_thread__get_ppid (struct pyrf_thread *pthread,void *closure__maybe_unused ) +{ +return PyLong_FromLong (thread__ppid (pthread->thread)); +} + +static PyObject *pyrf_thread__get_cpu (struct pyrf_thread *pthread,void *closure__maybe_unused ) +{ +return PyLong_FromLong (thread__cpu (pthread->thread)); +} + +static PyGetSetDef pyrf_thread__getset[] = { + { .name ="pid" , .get = (getter )pyrf_thread__get_pid, .doc ="process ID" }, + { .name ="tid" , .get = (getter )pyrf_thread__get_tid, .doc ="thread ID" }, + { .name ="ppid" , .get = (getter )pyrf_thread__get_ppid, .doc ="parent process ID" }, + { .name ="cpu" , .get = (getter )pyrf_thread__get_cpu, .doc ="cpu number" }, + { .name =NULL } +}; + +static const char pyrf_thread__doc[] =PyDoc_STR ("perf thread object." ); + +static PyTypeObject pyrf_thread__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.thread" , + .tp_basicsize =sizeof (struct pyrf_thread ), + .tp_dealloc = (destructor )pyrf_thread__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_methods = pyrf_thread__methods, + .tp_getset = pyrf_thread__getset, + .tp_doc = pyrf_thread__doc, +}; + +static int pyrf_thread__setup_types (void ) +{ +return PyType_Ready (&pyrf_thread__type); +} + +static PyObject *pyrf_thread__from_thread (struct thread *thread) +{ + struct pyrf_thread *pthread = PyObject_New(struct pyrf_thread, &pyrf_thread__type); + +if (!pthread) +return NULL ; + + pthread->thread =thread__get (thread); +return (PyObject *)pthread; +} + +struct pyrf_session { +PyObject_HEAD + + struct perf_session *session; +struct perf_tool tool; +struct pyrf_data *pdata; +PyObject *sample; +}; + +static int pyrf_session_tool__sample (const struct perf_tool *tool, +union perf_event *event, +struct perf_sample *sample, +struct machine *machine__maybe_unused ) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); +PyObject *pyevent =pyrf_event__new (event); +struct pyrf_event *pevent = (struct pyrf_event *)pyevent; +PyObject *ret; +bool needs_swap; + +if (pyevent ==NULL ) +return -ENOMEM ; + + needs_swap = sample->evsel->needs_swap; + +memcpy (&pevent->event, event, event->header.size); +/* The event is already swapped, disable swapping temporarily */ + sample->evsel->needs_swap = false; +if (evsel__parse_sample (sample->evsel, &pevent->event, &pevent->sample) <0 ) { + sample->evsel->needs_swap = needs_swap; +Py_DECREF (pyevent); +return -1 ; + } + sample->evsel->needs_swap = needs_swap; + + ret =PyObject_CallFunction (psession->sample,"O" , pyevent); +if (!ret) { +Py_DECREF (pyevent); +return -1 ; + } +Py_DECREF (ret); +Py_DECREF (pyevent); +return 0 ; +} + +static PyObject *pyrf_session__find_thread (struct pyrf_session *psession,PyObject *args) +{ +struct machine *machine; +struct thread *thread =NULL ; +PyObject *result; +int pid; + +CHECK_INITIALIZED (psession->session,"session" ); + +if (!PyArg_ParseTuple (args,"i" , &pid)) +return NULL ; + + machine = &psession->session->machines.host; + thread =machine__find_thread (machine, pid, pid); + +if (!thread) { + machine =perf_session__find_machine (psession->session, pid); +if (machine) + thread =machine__find_thread (machine, pid, pid); + } + +if (!thread) { +PyErr_Format (PyExc_TypeError,"Failed to find thread %d" , pid); +return NULL ; + } + result =pyrf_thread__from_thread (thread); +thread__put (thread); +return result; +} + +static PyObject *pyrf_session__new (PyTypeObject *type,PyObject *args,PyObject *kwargs) +{ +struct pyrf_data *pdata; +PyObject *sample =NULL ; +static char *kwlist[] = {"data" ,"sample" ,NULL }; +struct pyrf_session *psession; +struct perf_session *session; + +if (!PyArg_ParseTupleAndKeywords (args, kwargs,"O!|O" , kwlist, &pyrf_data__type, &pdata, + &sample)) +return NULL ; + + psession = PyObject_New(struct pyrf_session, type); +if (!psession) +return NULL ; + + psession->session =NULL ; + psession->sample =NULL ; + psession->pdata =NULL ; + +Py_INCREF (pdata); + psession->pdata = pdata; + +perf_tool__init (&psession->tool,/*ordered_events=*/ true); + psession->tool.ordering_requires_timestamps = true; + +#define ADD_TOOL (name) \ + do { \ + if (name) { \ + if (!PyCallable_Check(name)) { \ + PyErr_SetString(PyExc_TypeError, #name " must be callable"); \ + goto err_out; \ + } \ + psession->tool.name = pyrf_session_tool__##name; \ + Py_INCREF(name); \ + psession->name = name; \ + } \ + } while (0) + +ADD_TOOL (sample); +#undef ADD_TOOL + + psession->tool.comm = perf_event__process_comm; + psession->tool.mmap = perf_event__process_mmap; + psession->tool.mmap2 = perf_event__process_mmap2; + psession->tool.namespaces = perf_event__process_namespaces; + psession->tool.cgroup = perf_event__process_cgroup; + psession->tool.exit = perf_event__process_exit; + psession->tool.fork = perf_event__process_fork; + psession->tool.ksymbol = perf_event__process_ksymbol; + psession->tool.text_poke = perf_event__process_text_poke; + psession->tool.build_id = perf_event__process_build_id; + psession->tool.attr = perf_event__process_attr; + psession->tool.feature = perf_event__process_feature; + psession->tool.stat = perf_event__process_stat_event; + session =perf_session__new (&pdata->data, &psession->tool); +if (IS_ERR (session)) { +PyErr_Format (PyExc_IOError,"failed to create session: %ld" ,PTR_ERR (session)); + goto err_out; + } + psession->session = session; + +if (symbol__init (perf_session__env (session)) <0 ) { +PyErr_SetString (PyExc_OSError,"perf: symbol__init failed" ); + goto err_out; + } + + + +return (PyObject *)psession; +err_out: +Py_DECREF (psession); +return NULL ; +} + +static void pyrf_session__delete (struct pyrf_session *psession) +{ +perf_session__delete (psession->session); +Py_XDECREF (psession->pdata); +Py_XDECREF (psession->sample); +Py_TYPE (psession)->tp_free ((PyObject *)psession); +} + +static PyObject *pyrf_session__find_thread_events (struct pyrf_session *psession) +{ +int err; + +CHECK_INITIALIZED (psession->session,"session" ); + + err =perf_session__process_events (psession->session); + +if (PyErr_Occurred ()) +return NULL ; + +if (err <0 ) { +PyErr_Format (PyExc_OSError,"Process events failed: %d" , err); +return NULL ; + } + + Py_RETURN_NONE; +} + +static PyMethodDef pyrf_session__methods[] = { + { + .ml_name ="process_events" , + .ml_meth = (PyCFunction )pyrf_session__find_thread_events, + .ml_flags =METH_NOARGS , + .ml_doc =PyDoc_STR ("Iterate and process events." ) + }, + { + .ml_name ="find_thread" , + .ml_meth = (PyCFunction )pyrf_session__find_thread, + .ml_flags =METH_VARARGS , + .ml_doc =PyDoc_STR ("Returns the thread associated with a pid." ) + }, + { .ml_name =NULL , } +}; + +static const char pyrf_session__doc[] =PyDoc_STR ("perf session object." ); + +static PyObject *pyrf_session__getattro (struct pyrf_session *psession,PyObject *attr_name) +{ +if (!psession->session) { +PyErr_SetString (PyExc_ValueError,"session not initialized" ); +return NULL ; + } +return PyObject_GenericGetAttr ((PyObject *) psession, attr_name); +} + +static int pyrf_session__setattro (struct pyrf_session *psession,PyObject *attr_name, +PyObject *value) +{ +if (!psession->session) { +PyErr_SetString (PyExc_ValueError,"session not initialized" ); +return -1 ; + } +return PyObject_GenericSetAttr ((PyObject *) psession, attr_name, value); +} + +static PyTypeObject pyrf_session__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.session" , + .tp_basicsize =sizeof (struct pyrf_session ), + .tp_dealloc = (destructor )pyrf_session__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_methods = pyrf_session__methods, + .tp_doc = pyrf_session__doc, + .tp_new = pyrf_session__new, + .tp_getattro = (getattrofunc ) pyrf_session__getattro, + .tp_setattro = (setattrofunc ) pyrf_session__setattro, +}; + +static int pyrf_session__setup_types (void ) +{ +return PyType_Ready (&pyrf_session__type); +} +static PyMethodDef perf__methods[] = { { .ml_name ="metrics" ,
@@ -2633,8 +2967,10 @@ PyMODINIT_FUNC PyInit_perf(void) };PyObject *module =PyModule_Create (&moduledef); - if (module == NULL || - pyrf_event__setup_types() < 0 || +if (module ==NULL ) +return NULL ; + + if (pyrf_event__setup_types() < 0 || pyrf_evlist__setup_types() < 0 || pyrf_evsel__setup_types() < 0 || pyrf_thread_map__setup_types() < 0 ||
@@ -2642,8 +2978,12 @@ PyMODINIT_FUNC PyInit_perf(void) pyrf_pmu_iterator__setup_types() < 0 || pyrf_pmu__setup_types() < 0 || pyrf_counts_values__setup_types() < 0 || - pyrf_data__setup_types() < 0) - return module; + pyrf_data__setup_types() < 0 || + pyrf_session__setup_types() < 0 || + pyrf_thread__setup_types() < 0) { +Py_DECREF (module); +return NULL ; + }/* The page_size is placed in util object. */ page_size =sysconf (_SC_PAGE_SIZE);
@@ -2693,6 +3033,9 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_data__type);PyModule_AddObject (module,"data" , (PyObject *)&pyrf_data__type); +Py_INCREF (&pyrf_session__type); +PyModule_AddObject (module,"session" , (PyObject *)&pyrf_session__type); + dict =PyModule_GetDict (module);if (dict ==NULL ) goto error;
@@ -2706,7 +3049,9 @@ PyMODINIT_FUNC PyInit_perf(void) } error: -if (PyErr_Occurred ()) -PyErr_SetString (PyExc_ImportError,"perf: Init failed!" ); +if (PyErr_Occurred ()) { +Py_XDECREF (module); +return NULL ; + }return module; }
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 10/21] perf python: Add python session abstraction wrapping perf's session
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [High] Cross-endian perf.data parsing is broken for 32-bit fields (pid, tid, cpu) because `needs_swap` is…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] Cross-endian perf.data parsing is broken for 32-bit fields (pid, tid, cpu) because needs_swap is temporarily forced to false. --
commit d6d1f3c58a23e62a794f6e4127e3eb5d94ce2db6
Author: Ian Rogers <[email protected]>
perf python: Add python session abstraction wrapping perf's session
This commit introduces a Python wrapper for perf_session, allowing perf.data files to be processed natively within Python scripts. It adds support for a sample callback during event processing and exposes thread information for querying.
Quoted 4 lines
tools/perf/util/python.c +0
-0
[ ... ]
Quoted 4 lines
tools/perf/util/python.c +0
-0@@ -2579,6 +2587,332 @@ static int pyrf_data__setup_types(void)return PyType_Ready (&pyrf_data__type); }
[ ... ]
Quoted 10 lines
tools/perf/util/python.c +10
-0+static int pyrf_session_tool__sample (const struct perf_tool *tool, +union perf_event *event, +struct perf_sample *sample, +struct machine *machine__maybe_unused ) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); +PyObject *pyevent =pyrf_event__new (event); +struct pyrf_event *pevent = (struct pyrf_event *)pyevent; +PyObject *ret; +bool needs_swap;
tools/perf/util/python.c static int pyrf_data__setup_types(void) · line 2712
Quoted 10 lines
tools/perf/util/python.c +10
-0+ +if (pyevent ==NULL ) +return -ENOMEM ; + + needs_swap = sample->evsel->needs_swap; + +memcpy (&pevent->event, event, event->header.size); +/* The event is already swapped, disable swapping temporarily */ + sample->evsel->needs_swap = false; +if (evsel__parse_sample (sample->evsel, &pevent->event, &pevent->sample) <0 ) {
[Severity: High]
Does disabling needs_swap here break cross-endian parsing for 32-bit fields like pid, tid, and cpu?
When processing a cross-endian perf.data file, the core perf framework first swaps the entire payload as an array of 64-bit integers. Because 32-bit fields are packed into a 64-bit slot, this generic 64-bit swap reverses both their byte order and their relative positions.
It looks like evsel__parse_sample() relies on needs_swap being true so it can un-swap the 64-bit value and apply bswap_32 to the individual components. By temporarily forcing it to false, the Python tool might silently read garbage values for these fields during cross-endian analysis.
Quoted 15 lines
tools/perf/util/python.c +15
-0+ sample->evsel->needs_swap = needs_swap; +Py_DECREF (pyevent); +return -1 ; + } + sample->evsel->needs_swap = needs_swap; + + ret =PyObject_CallFunction (psession->sample,"O" , pyevent); +if (!ret) { +Py_DECREF (pyevent); +return -1 ; + } +Py_DECREF (ret); +Py_DECREF (pyevent); +return 0 ; +}
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=10
Ian Rogers
reviewerPATCH v21 11/21 perf python: Refactor and add accessors to sample event
In reply to Ian RogersAdd common evsel field for events and move sample specific fields to only be present in sample events. Add accessors for sample events. Ensure offsets are within the bounds of the event. Allocate just enough memory for the copied event, don't make the maximum event size each time.
--- tools/perf/util/python.c | 519 +++++++++++++++++++++++++++++++------- tools/perf/util/session.c | 2 +- tools/perf/util/session.h | 2 + 3 files changed, 429 insertions(+), 94 deletions(-)
tools/perf/util/python.c +426 -93
@@ -13,21 +13,28 @@#include <perf/mmap.h> #include <structmember.h> +#include "addr_location.h" +#include "build-id.h" #include "callchain.h" #include "comm.h" #include "counts.h" #include "data.h" #include "debug.h" +#include "dso.h" #include "event.h" #include "evlist.h" #include "evsel.h" #include "expr.h" +#include "map.h" #include "metricgroup.h" #include "mmap.h" #include "pmus.h" #include "print_binary.h" #include "record.h" +#include "sample.h" #include "session.h" +#include "srccode.h" +#include "srcline.h" #include "strbuf.h" #include "symbol.h" #include "stat.h"
@@ -37,7 +44,6 @@#include "tool.h" #include "tp_pmu.h" #include "trace-event.h" -#include "util/sample.h" #ifdef HAVE_LIBTRACEEVENT #include <event-parse.h>
@@ -45,6 +51,8 @@PyMODINIT_FUNC PyInit_perf (void ); +static PyObject *pyrf_evsel__from_evsel (struct evsel *evsel); +#define member_def (type, member, ptype, help) \ { #member, ptype, \ offsetof(struct pyrf_event, event) + offsetof(struct type, member), \
@@ -73,21 +81,53 @@ PyMODINIT_FUNC PyInit_perf(void);struct pyrf_event {PyObject_HEAD +/** @sample: The parsed sample from the event. */ struct perf_sample sample; -union perf_event event; +/** @al: The address location from machine__resolve, lazily computed. */ +struct addr_location al; +/** @al_resolved: True when machine__resolve been called. */ +bool al_resolved; +/** @event: The underlying perf_event that may be in a file or ring buffer. */ +union perf_event event; };#define sample_members \ - sample_member_def(sample_ip, ip, T_ULONGLONG, "event ip"), \ sample_member_def(sample_pid, pid, T_INT, "event pid"), \ sample_member_def(sample_tid, tid, T_INT, "event tid"), \ sample_member_def(sample_time, time, T_ULONGLONG, "event timestamp"), \ - sample_member_def(sample_addr, addr, T_ULONGLONG, "event addr"), \ sample_member_def(sample_id, id, T_ULONGLONG, "event id"), \ sample_member_def(sample_stream_id, stream_id, T_ULONGLONG, "event stream id"), \ sample_member_def(sample_period, period, T_ULONGLONG, "event period"), \ sample_member_def(sample_cpu, cpu, T_UINT, "event cpu"), +static PyObject *pyrf_event__get_evsel (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (!pevent->sample.evsel) + Py_RETURN_NONE; + +return pyrf_evsel__from_evsel (pevent->sample.evsel); +} + +static PyGetSetDef pyrf_event__getset[] = { + { + .name ="evsel" , + .get = pyrf_event__get_evsel, + .set =NULL , + .doc ="tracking event." , + }, + { .name =NULL , }, +}; + +static void pyrf_event__delete (struct pyrf_event *pevent) +{ +if (pevent->al_resolved) +addr_location__exit (&pevent->al); +perf_sample__exit (&pevent->sample); +Py_TYPE (pevent)->tp_free ((PyObject *)pevent); +} +static const char pyrf_mmap_event__doc[] =PyDoc_STR ("perf mmap event object." );static PyMemberDef pyrf_mmap_event__members[] = {
@@ -126,9 +166,11 @@ static PyTypeObject pyrf_mmap_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.mmap_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_mmap_event__doc, .tp_members = pyrf_mmap_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_mmap_event__repr, };
@@ -161,9 +203,11 @@ static PyTypeObject pyrf_task_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.task_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_task_event__doc, .tp_members = pyrf_task_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_task_event__repr, };
@@ -190,9 +234,11 @@ static PyTypeObject pyrf_comm_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.comm_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_comm_event__doc, .tp_members = pyrf_comm_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_comm_event__repr, };
@@ -222,9 +268,11 @@ static PyTypeObject pyrf_throttle_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.throttle_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_throttle_event__doc, .tp_members = pyrf_throttle_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_throttle_event__repr, };
@@ -232,6 +280,7 @@ static const char pyrf_lost_event__doc[] = PyDoc_STR("perf lost event object.");static PyMemberDef pyrf_lost_event__members[] = { sample_members +member_def (perf_event_header, type,T_UINT ,"event type" ),member_def (perf_record_lost, id,T_ULONGLONG ,"event id" ),member_def (perf_record_lost, lost,T_ULONGLONG ,"number of lost events" ), { .name =NULL , },
@@ -257,9 +306,11 @@ static PyTypeObject pyrf_lost_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.lost_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_lost_event__doc, .tp_members = pyrf_lost_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_lost_event__repr, };
@@ -267,6 +318,7 @@ static const char pyrf_read_event__doc[] = PyDoc_STR("perf read event object.");static PyMemberDef pyrf_read_event__members[] = { sample_members +member_def (perf_event_header, type,T_UINT ,"event type" ),member_def (perf_record_read, pid,T_UINT ,"event pid" ),member_def (perf_record_read, tid,T_UINT ,"event tid" ), { .name =NULL , },
@@ -287,9 +339,11 @@ static PyTypeObject pyrf_read_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.read_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_read_event__doc, .tp_members = pyrf_read_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_read_event__repr, };
@@ -297,16 +351,17 @@ static const char pyrf_sample_event__doc[] = PyDoc_STR("perf sample event objectstatic PyMemberDef pyrf_sample_event__members[] = { sample_members +sample_member_def (sample_ip, ip,T_ULONGLONG ,"event ip" ), +sample_member_def (sample_addr, addr,T_ULONGLONG ,"event addr" ), +sample_member_def (sample_phys_addr, phys_addr,T_ULONGLONG ,"event physical addr" ), +sample_member_def (sample_weight, weight,T_ULONGLONG ,"event weight" ), +sample_member_def (sample_data_src, data_src,T_ULONGLONG ,"event data source" ), +sample_member_def (sample_insn_count, insn_cnt,T_ULONGLONG ,"event instruction count" ), +sample_member_def (sample_cyc_count, cyc_cnt,T_ULONGLONG ,"event cycle count" ),member_def (perf_event_header, type,T_UINT ,"event type" ), { .name =NULL , }, }; -static void pyrf_sample_event__delete (struct pyrf_event *pevent) -{ -perf_sample__exit (&pevent->sample); -Py_TYPE (pevent)->tp_free ((PyObject *)pevent); -} -static PyObject *pyrf_sample_event__repr (const struct pyrf_event *pevent) {PyObject *ret;
@@ -324,6 +379,8 @@ static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent)#ifdef HAVE_LIBTRACEEVENT static bool is_tracepoint (const struct pyrf_event *pevent) { +if (!pevent->sample.evsel) +return false;return pevent->sample.evsel->core.attr.type ==PERF_TYPE_TRACEPOINT ; }
@@ -394,6 +451,199 @@ get_tracepoint_field(struct pyrf_event *pevent, PyObject *attr_name) }#endif /* HAVE_LIBTRACEEVENT */ +static int pyrf_sample_event__resolve_al (struct pyrf_event *pevent) +{ +struct evsel *evsel = pevent->sample.evsel; +struct evlist *evlist = evsel ? evsel->evlist :NULL ; +struct perf_session *session = evlist ?evlist__session (evlist) :NULL ; + +if (pevent->al_resolved) +return 0 ; + +if (!session) +return -1 ; + +addr_location__init (&pevent->al); +if (machine__resolve (&session->machines.host, &pevent->al, &pevent->sample) <0 ) { +addr_location__exit (&pevent->al); +return -1 ; + } + + pevent->al_resolved = true; +return 0 ; +} + +static PyObject *pyrf_sample_event__get_dso (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +return PyUnicode_FromString (dso__name (map__dso (pevent->al.map))); +} + +static PyObject *pyrf_sample_event__get_dso_long_name (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +return PyUnicode_FromString (dso__long_name (map__dso (pevent->al.map))); +} + +static PyObject *pyrf_sample_event__get_dso_bid (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +char sbuild_id[SBUILD_ID_SIZE ]; + +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +build_id__snprintf (dso__bid (map__dso (pevent->al.map)), sbuild_id,sizeof (sbuild_id)); +return PyUnicode_FromString (sbuild_id); +} + +static PyObject *pyrf_sample_event__get_map_start (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLong (map__start (pevent->al.map)); +} + +static PyObject *pyrf_sample_event__get_map_end (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLong (map__end (pevent->al.map)); +} + +static PyObject *pyrf_sample_event__get_map_pgoff (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.map) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLongLong (map__pgoff (pevent->al.map)); +} + +static PyObject *pyrf_sample_event__get_symbol (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.sym) + Py_RETURN_NONE; + +return PyUnicode_FromString (pevent->al.sym->name); +} + +static PyObject *pyrf_sample_event__get_sym_start (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.sym) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLongLong (pevent->al.sym->start); +} + +static PyObject *pyrf_sample_event__get_sym_end (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pyrf_sample_event__resolve_al (pevent) <0 || !pevent->al.sym) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLongLong (pevent->al.sym->end); +} + +static PyObject *pyrf_sample_event__get_raw_buf (struct pyrf_event *pevent, +void *closure__maybe_unused ) +{ +if (pevent->event.header.type !=PERF_RECORD_SAMPLE ) + Py_RETURN_NONE; + +return PyBytes_FromStringAndSize ((const char *)pevent->sample.raw_data, + pevent->sample.raw_size); +} + +static PyObject *pyrf_sample_event__srccode (PyObject *self,PyObject *args) +{ +struct pyrf_event *pevent = (void *)self; +u64 addr = pevent->sample.ip; +char *srcfile =NULL ; +char *srccode =NULL ; +unsigned int line =0 ; +int len =0 ; +PyObject *result; +struct addr_location al; + +if (!PyArg_ParseTuple (args,"|K" , &addr)) +return NULL ; + +if (pyrf_sample_event__resolve_al (pevent) <0 ) + Py_RETURN_NONE; + +if (addr != pevent->sample.ip) { +addr_location__init (&al); +thread__find_symbol_fb (pevent->al.thread, pevent->sample.cpumode, addr, &al); + }else { +addr_location__init (&al); + al.thread =thread__get (pevent->al.thread); + al.map =map__get (pevent->al.map); + al.sym = pevent->al.sym; + al.addr = pevent->al.addr; + } + +if (al.map) { +struct dso *dso =map__dso (al.map); + +if (dso) { + srcfile =get_srcline_split (dso,map__rip_2objdump (al.map, addr), + &line); + } + } +addr_location__exit (&al); + +if (srcfile) { + srccode =find_sourceline (srcfile, line, &len); + result =Py_BuildValue ("(sIs#)" , srcfile, line, srccode, (Py_ssize_t )len); +free (srcfile); + }else { + result =Py_BuildValue ("(sIs#)" ,NULL ,0 ,NULL , (Py_ssize_t )0 ); + } + +return result; +} + +static PyObject *pyrf_sample_event__insn (PyObject *self,PyObject *args__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; +struct thread *thread; +struct machine *machine; + +if (pyrf_sample_event__resolve_al (pevent) <0 ) + Py_RETURN_NONE; + + thread = pevent->al.thread; + +if (!thread || !thread__maps (thread)) + Py_RETURN_NONE; + + machine =maps__machine (thread__maps (thread)); +if (!machine) + Py_RETURN_NONE; + +if (pevent->sample.ip && !pevent->sample.insn_len) +perf_sample__fetch_insn (&pevent->sample, thread, machine); + +if (!pevent->sample.insn_len) + Py_RETURN_NONE; + +return PyBytes_FromStringAndSize ((const char *)pevent->sample.insn, + pevent->sample.insn_len); +} +static PyObject *pyrf_sample_event__getattro (struct pyrf_event *pevent,PyObject *attr_name) {
@@ -407,13 +657,102 @@ pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name)return obj ?:PyObject_GenericGetAttr ((PyObject *) pevent, attr_name); } +static PyGetSetDef pyrf_sample_event__getset[] = { + { + .name ="raw_buf" , + .get = (getter )pyrf_sample_event__get_raw_buf, + .set =NULL , + .doc ="event raw buffer." , + }, + { + .name ="evsel" , + .get = pyrf_event__get_evsel, + .set =NULL , + .doc ="tracking event." , + }, + { + .name ="dso" , + .get = (getter )pyrf_sample_event__get_dso, + .set =NULL , + .doc ="event dso short name." , + }, + { + .name ="dso_long_name" , + .get = (getter )pyrf_sample_event__get_dso_long_name, + .set =NULL , + .doc ="event dso long name." , + }, + { + .name ="dso_bid" , + .get = (getter )pyrf_sample_event__get_dso_bid, + .set =NULL , + .doc ="event dso build id." , + }, + { + .name ="map_start" , + .get = (getter )pyrf_sample_event__get_map_start, + .set =NULL , + .doc ="event map start address." , + }, + { + .name ="map_end" , + .get = (getter )pyrf_sample_event__get_map_end, + .set =NULL , + .doc ="event map end address." , + }, + { + .name ="map_pgoff" , + .get = (getter )pyrf_sample_event__get_map_pgoff, + .set =NULL , + .doc ="event map page offset." , + }, + { + .name ="symbol" , + .get = (getter )pyrf_sample_event__get_symbol, + .set =NULL , + .doc ="event symbol name." , + }, + { + .name ="sym_start" , + .get = (getter )pyrf_sample_event__get_sym_start, + .set =NULL , + .doc ="event symbol start address." , + }, + { + .name ="sym_end" , + .get = (getter )pyrf_sample_event__get_sym_end, + .set =NULL , + .doc ="event symbol end address." , + }, + { .name =NULL , }, +}; + +static PyMethodDef pyrf_sample_event__methods[] = { + { + .ml_name ="srccode" , + .ml_meth = (PyCFunction )pyrf_sample_event__srccode, + .ml_flags =METH_VARARGS , + .ml_doc =PyDoc_STR ("Get source code for an address." ) + }, + { + .ml_name ="insn" , + .ml_meth = (PyCFunction )pyrf_sample_event__insn, + .ml_flags =METH_NOARGS , + .ml_doc =PyDoc_STR ("Get instruction bytes for a sample." ) + }, + { .ml_name =NULL , } +}; +static PyTypeObject pyrf_sample_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name ="perf.sample_event" , .tp_basicsize =sizeof (struct pyrf_event ), + .tp_dealloc = (destructor )pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_sample_event__doc, .tp_members = pyrf_sample_event__members, + .tp_getset = pyrf_sample_event__getset, + .tp_methods = pyrf_sample_event__methods, .tp_repr = (reprfunc )pyrf_sample_event__repr, .tp_getattro = (getattrofunc ) pyrf_sample_event__getattro, };
@@ -449,25 +788,17 @@ static PyTypeObject pyrf_context_switch_event__type = {PyVarObject_HEAD_INIT (NULL ,0 ) .tp_name = "perf.context_switch_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_context_switch_event__doc, .tp_members = pyrf_context_switch_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_context_switch_event__repr, };static int pyrf_event__setup_types (void ) {int err; - pyrf_mmap_event__type.tp_new = - pyrf_task_event__type.tp_new = - pyrf_comm_event__type.tp_new = - pyrf_lost_event__type.tp_new = - pyrf_read_event__type.tp_new = - pyrf_sample_event__type.tp_new = - pyrf_context_switch_event__type.tp_new = - pyrf_throttle_event__type.tp_new = PyType_GenericNew; - - pyrf_sample_event__type.tp_dealloc = (destructor )pyrf_sample_event__delete, err =PyType_Ready (&pyrf_mmap_event__type);if (err <0 )
@@ -511,33 +842,57 @@ static PyTypeObject *pyrf_event__type[] = { [PERF_RECORD_SWITCH_CPU_WIDE] = &pyrf_context_switch_event__type, }; -static PyObject *pyrf_event__new (const union perf_event *event) +static PyObject *pyrf_event__new (const union perf_event *event,struct evsel *evsel) {struct pyrf_event *pevent; -PyTypeObject *ptype; +int err; +u32 min_size; +bool needs_swap; -if ((event->header.type <PERF_RECORD_MMAP || - event->header.type >PERF_RECORD_SAMPLE ) && - !(event->header.type ==PERF_RECORD_SWITCH || - event->header.type ==PERF_RECORD_SWITCH_CPU_WIDE )) { -PyErr_Format (PyExc_TypeError,"Unexpected header type %u" , +if (event->header.type >=ARRAY_SIZE (pyrf_event__type) || + pyrf_event__type[event->header.type] ==NULL ) { +return PyErr_Format (PyExc_TypeError,"Unexpected header type %u" , event->header.type); -return NULL ; } -// FIXME this better be dynamic or we need to parse everything -// before calling perf_mmap__consume(), including tracepoint fields. -if (sizeof (pevent->event) < event->header.size) { -PyErr_Format (PyExc_TypeError,"Unexpected event size: %zd < %u" , -sizeof (pevent->event), event->header.size); -return NULL ; +if (perf_event__too_small (event, &min_size)) { +return PyErr_Format (PyExc_ValueError,"Event size %u too small for type %u" , + event->header.size, event->header.type); } - ptype = pyrf_event__type[event->header.type]; - pevent = PyObject_New(struct pyrf_event, ptype); -if (pevent !=NULL ) { -memcpy (&pevent->event, event, event->header.size); -perf_sample__init (&pevent->sample,/*all=*/ false); +size_t copy_size = event->header.size; + +if (copy_size >sizeof (pevent->event)) { +return PyErr_Format (PyExc_TypeError,"Unexpected event size: %zd < %zu" , +sizeof (pevent->event), copy_size); + } + + pevent = PyObject_New(struct pyrf_event, pyrf_event__type[event->header.type]); +if (pevent ==NULL ) +return PyErr_NoMemory (); + +/* Copy the event for memory safety and initialize variables. */ +memcpy (&pevent->event, event, copy_size); +if (copy_size <sizeof (pevent->event)) +memset ((char *)&pevent->event + copy_size,0 ,sizeof (pevent->event) - copy_size); + +perf_sample__init (&pevent->sample,/*all=*/ true); + pevent->al_resolved = false; +addr_location__init (&pevent->al); + +if (!evsel) +return (PyObject *)pevent; + +/* Parse the sample again so that pointers are within the copied event. */ + needs_swap = evsel->needs_swap; + + evsel->needs_swap = false; + err =evsel__parse_sample (evsel, &pevent->event, &pevent->sample); + evsel->needs_swap = needs_swap; +if (err <0 ) { +Py_DECREF (pevent); +return PyErr_Format (PyExc_OSError, +"perf: can't parse sample, err=%d" , err); }return (PyObject *)pevent; }
@@ -1244,7 +1599,7 @@ static PyObject *pyrf_evsel__str(PyObject *self)if (!evsel)return PyUnicode_FromString ("evsel(uninitialized)" ); -return PyUnicode_FromFormat ("evsel(%s/%s/)" ,evsel__pmu_name (evsel),evsel__name (evsel)); +return PyUnicode_FromFormat ("evsel(%s)" ,evsel__name (evsel)); }static PyMethodDef pyrf_evsel__methods[] = {
@@ -1878,9 +2233,11 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, {struct evlist *evlist;union perf_event *event; +struct evsel *evsel;int sample_id_all =1 , cpu;static char *kwlist[] = {"cpu" ,"sample_id_all" ,NULL };struct mmap *md; +PyObject *pyevent;int err;CHECK_INITIALIZED (pevlist->evlist,"evlist" );
@@ -1891,44 +2248,33 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist,return NULL ; md =get_md (evlist, cpu); -if (!md) { -PyErr_Format (PyExc_TypeError,"Unknown CPU '%d'" , cpu); -return NULL ; - } +if (!md) +return PyErr_Format (PyExc_TypeError,"Unknown CPU '%d'" , cpu); -if (perf_mmap__read_init (&md->core) <0 ) - goto end; + err =perf_mmap__read_init (&md->core); +if (err <0 ) { +if (err == -EAGAIN ) + Py_RETURN_NONE; +return PyErr_Format (PyExc_OSError, +"perf: error mmap read init, err=%d" , err); + } event =perf_mmap__read_event (&md->core); -if (event !=NULL ) { -PyObject *pyevent =pyrf_event__new (event); -struct pyrf_event *pevent = (struct pyrf_event *)pyevent; -struct evsel *evsel; - -if (pyevent ==NULL ) -return PyErr_NoMemory (); - - evsel =evlist__event2evsel (evlist, event); -if (!evsel) { -Py_DECREF (pyevent); -Py_INCREF (Py_None); -return Py_None; - } +if (event ==NULL ) + Py_RETURN_NONE; + evsel =evlist__event2evsel (evlist, event); +if (!evsel) { +/* Unknown evsel. */ perf_mmap__consume (&md->core); - - err =evsel__parse_sample (evsel, &pevent->event, &pevent->sample); -if (err) { -Py_DECREF (pyevent); -return PyErr_Format (PyExc_OSError, -"perf: can't parse sample, err=%d" , err); - } - -return pyevent; + Py_RETURN_NONE; } -end: -Py_INCREF (Py_None); -return Py_None; + pyevent =pyrf_event__new (event, evsel); +perf_mmap__consume (&md->core); +if (pyevent ==NULL ) +return PyErr_Occurred () ?NULL :PyErr_NoMemory (); + +return pyevent; } static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist,
@@ -2140,10 +2486,7 @@ static PyObject *pyrf_evlist__str(PyObject *self)evlist__for_each_entry (pevlist->evlist, pos) {if (!first)strbuf_addch (&sb,',' ); -if (!pos->pmu) -strbuf_addstr (&sb,evsel__name (pos)); -else -strbuf_addf (&sb,"%s/%s/" , pos->pmu->name,evsel__name (pos)); +strbuf_addstr (&sb,evsel__name (pos)); first = false; }strbuf_addstr (&sb,"])" );
@@ -2225,6 +2568,7 @@ static const struct perf_constant perf__constants[] = {PERF_CONST (TYPE_BREAKPOINT ),PERF_CONST (COUNT_HW_CPU_CYCLES ), +PERF_CONST (COUNT_HW_REF_CPU_CYCLES ),PERF_CONST (COUNT_HW_INSTRUCTIONS ),PERF_CONST (COUNT_HW_CACHE_REFERENCES ), PERF_CONST(COUNT_HW_CACHE_MISSES),
@@ -2696,26 +3040,12 @@ static int pyrf_session_tool__sample(const struct perf_tool *tool, struct machine *machine __maybe_unused) { struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); -PyObject *pyevent =pyrf_event__new (event); -struct pyrf_event *pevent = (struct pyrf_event *)pyevent; +PyObject *pyevent =pyrf_event__new (event, sample->evsel);PyObject *ret; -bool needs_swap;if (pyevent ==NULL )return -ENOMEM ; - needs_swap = sample->evsel->needs_swap; - -memcpy (&pevent->event, event, event->header.size); -/* The event is already swapped, disable swapping temporarily */ - sample->evsel->needs_swap = false; -if (evsel__parse_sample (sample->evsel, &pevent->event, &pevent->sample) <0 ) { - sample->evsel->needs_swap = needs_swap; -Py_DECREF (pyevent); -return -1 ; - } - sample->evsel->needs_swap = needs_swap; - ret =PyObject_CallFunction (psession->sample,"O" , pyevent);if (!ret) {Py_DECREF (pyevent);
@@ -2989,10 +3319,13 @@ PyMODINIT_FUNC PyInit_perf(void) page_size =sysconf (_SC_PAGE_SIZE);Py_INCREF (&pyrf_evlist__type); -PyModule_AddObject (module,"evlist" , (PyObject *)&pyrf_evlist__type); +PyModule_AddObject (module,"evlist" , (PyObject *)&pyrf_evlist__type);Py_INCREF (&pyrf_evsel__type); -PyModule_AddObject (module,"evsel" , (PyObject *)&pyrf_evsel__type); +PyModule_AddObject (module,"evsel" , (PyObject *)&pyrf_evsel__type); + +Py_INCREF (&pyrf_thread__type); +PyModule_AddObject (module,"thread" , (PyObject *)&pyrf_thread__type);Py_INCREF (&pyrf_mmap_event__type);PyModule_AddObject (module,"mmap_event" , (PyObject *)&pyrf_mmap_event__type);
tools/perf/util/session.c +1 -1
@@ -2672,7 +2672,7 @@ static const u32 perf_event__min_size[PERF_RECORD_HEADER_MAX] = {* Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX. * If min is non-NULL, stores the required minimum on failure. */ -static bool perf_event__too_small (const union perf_event *event,u32 *min) +bool perf_event__too_small (const union perf_event *event,u32 *min) {u32 min_sz = perf_event__min_size[event->header.type];
tools/perf/util/session.h +2 -0
@@ -122,6 +122,8 @@ void perf_session__delete(struct perf_session *session);void perf_event_header__bswap (struct perf_event_header *hdr); +bool perf_event__too_small (const union perf_event *event,u32 *min); + int perf_session__peek_event(struct perf_session *session, off_t file_offset, void *buf, size_t buf_sz, union perf_event **event_ptr,
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 11/21] perf python: Refactor and add accessors to sample event
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Medium] The code fails to realize the commit message's claim to 'Allocate just enough memory for the…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The code fails to realize the commit message's claim to 'Allocate just enough memory for the copied event, don't make the maximum event size each time'. --
commit 72cd19fc97123aafbff114427cd0ae63ceaea3bd
Author: Ian Rogers <[email protected]>
perf python: Refactor and add accessors to sample event
This commit refactors event sample specific fields to only be present in sample events and adds accessors for them. It also ensures offsets are bounds checked and attempts to optimize the memory allocated for the copied event.
Quoted 4 lines
tools/perf/util/python.c +0
-0
[ ... ]
Quoted 1 line
tools/perf/util/python.c +0
-0@@ -842,33 +842,57 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *ev
[ ... ]
tools/perf/util/python.c static PyTypeObject *pyrf_event__type[] = { · line 877
Quoted 8 lines
tools/perf/util/python.c +8
-0+ pevent = PyObject_New(struct pyrf_event, pyrf_event__type[event->header.type]); +if (pevent ==NULL ) +return PyErr_NoMemory (); + +/* Copy the event for memory safety and initialize variables. */ +memcpy (&pevent->event, event, copy_size); +if (copy_size <sizeof (pevent->event)) +memset ((char *)&pevent->event + copy_size,0 ,sizeof (pevent->event) - copy_size);
[Severity: Medium]
The commit message states "Allocate just enough memory for the copied event, don't make the maximum event size each time." Does this PyObject_New call actually reduce the allocation size?
Since PyObject_New relies on the statically defined tp_basicsize of the Python type (which is sizeof(struct pyrf_event)), and struct pyrf_event embeds union perf_event by value, the struct size remains constant. This means the allocation always reserves the maximum event size regardless of copy_size.
Is there a missing dynamic variable-sized allocation step, or should the commit message be updated to reflect that only the initialization footprint was modified?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=11
Ian Rogers
reviewerPATCH v21 12/21 perf python: Add mmap2 event
In reply to Ian RogersIf mmap is handled so should mmap2 events. Add support as a distinct python event type.
--- tools/perf/util/python.c | 153 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-)
tools/perf/util/python.c +152 -1
@@ -133,7 +133,7 @@ static const char pyrf_mmap_event__doc[] = PyDoc_STR("perf mmap event object.");static PyMemberDef pyrf_mmap_event__members[] = { sample_membersmember_def (perf_event_header, type,T_UINT ,"event type" ), -member_def (perf_event_header, misc,T_UINT ,"event misc" ), +member_def (perf_event_header, misc,T_USHORT ,"event misc" ),member_def (perf_record_mmap, pid,T_UINT ,"event pid" ),member_def (perf_record_mmap, tid,T_UINT ,"event tid" ),member_def (perf_record_mmap, start,T_ULONGLONG ,"start of the map" ),
@@ -174,6 +174,147 @@ static PyTypeObject pyrf_mmap_event__type = { .tp_repr = (reprfunc)pyrf_mmap_event__repr, }; +static const char pyrf_mmap2_event__doc[] =PyDoc_STR ("perf mmap2 event object." ); + +static PyObject *pyrf_mmap2_event__get_maj (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (pevent->event.header.misc &PERF_RECORD_MISC_MMAP_BUILD_ID ) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLong (pevent->event.mmap2.maj); +} + +static PyObject *pyrf_mmap2_event__get_min (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (pevent->event.header.misc &PERF_RECORD_MISC_MMAP_BUILD_ID ) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLong (pevent->event.mmap2.min); +} + +static PyObject *pyrf_mmap2_event__get_ino (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (pevent->event.header.misc &PERF_RECORD_MISC_MMAP_BUILD_ID ) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLongLong (pevent->event.mmap2.ino); +} + +static PyObject *pyrf_mmap2_event__get_ino_generation (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (pevent->event.header.misc &PERF_RECORD_MISC_MMAP_BUILD_ID ) + Py_RETURN_NONE; + +return PyLong_FromUnsignedLongLong (pevent->event.mmap2.ino_generation); +} + +static PyObject *pyrf_mmap2_event__get_build_id (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (!(pevent->event.header.misc &PERF_RECORD_MISC_MMAP_BUILD_ID )) + Py_RETURN_NONE; + +int size = pevent->event.mmap2.build_id_size; + +if (size >20 ) + size =20 ; + +return PyBytes_FromStringAndSize ((const char *)pevent->event.mmap2.build_id, size); +} + +static PyGetSetDef pyrf_mmap2_event__getset[] = { + { + .name ="evsel" , + .get = pyrf_event__get_evsel, + .set =NULL , + .doc ="tracking event." , + }, + { + .name ="maj" , + .get = pyrf_mmap2_event__get_maj, + .set =NULL , + .doc ="major number." , + }, + { + .name ="min" , + .get = pyrf_mmap2_event__get_min, + .set =NULL , + .doc ="minor number." , + }, + { + .name ="ino" , + .get = pyrf_mmap2_event__get_ino, + .set =NULL , + .doc ="inode number." , + }, + { + .name ="ino_generation" , + .get = pyrf_mmap2_event__get_ino_generation, + .set =NULL , + .doc ="inode generation." , + }, + { + .name ="build_id" , + .get = pyrf_mmap2_event__get_build_id, + .set =NULL , + .doc ="binary build ID." , + }, + { .name =NULL , }, +}; + +static PyMemberDef pyrf_mmap2_event__members[] = { + sample_members +member_def (perf_event_header, type,T_UINT ,"event type" ), +member_def (perf_event_header, misc,T_USHORT ,"event misc" ), +member_def (perf_record_mmap2, pid,T_UINT ,"event pid" ), +member_def (perf_record_mmap2, tid,T_UINT ,"event tid" ), +member_def (perf_record_mmap2, start,T_ULONGLONG ,"start of the map" ), +member_def (perf_record_mmap2, len,T_ULONGLONG ,"map length" ), +member_def (perf_record_mmap2, pgoff,T_ULONGLONG ,"page offset" ), +member_def (perf_record_mmap2, prot,T_UINT ,"protection" ), +member_def (perf_record_mmap2, flags,T_UINT ,"flags" ), +member_def (perf_record_mmap2, filename,T_STRING_INPLACE ,"backing store" ), + { .name =NULL , }, +}; + +static PyObject *pyrf_mmap2_event__repr (const struct pyrf_event *pevent) +{ +PyObject *ret; +char *s; + +if (asprintf (&s,"{ type: mmap2, pid: %u, tid: %u, start: %#" PRI_lx64", length: %#" PRI_lx64", offset: %#" PRI_lx64", flags: %#x, prot: %#x, filename: %s }" , + pevent->event.mmap2.pid, pevent->event.mmap2.tid, + pevent->event.mmap2.start, pevent->event.mmap2.len, + pevent->event.mmap2.pgoff, pevent->event.mmap2.flags, + pevent->event.mmap2.prot, pevent->event.mmap2.filename) <0 ) +return PyErr_NoMemory (); + + ret =PyUnicode_FromString (s); +free (s); +return ret; +} + +static PyTypeObject pyrf_mmap2_event__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.mmap2_event" , + .tp_basicsize =sizeof (struct pyrf_event ), + .tp_dealloc = (destructor )pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_mmap2_event__doc, + .tp_members = pyrf_mmap2_event__members, + .tp_getset = pyrf_mmap2_event__getset, + .tp_repr = (reprfunc )pyrf_mmap2_event__repr, +}; +static const char pyrf_task_event__doc[] =PyDoc_STR ("perf task (fork/exit) event object." );static PyMemberDef pyrf_task_event__members[] = {
@@ -801,6 +942,9 @@ static int pyrf_event__setup_types(void)int err; err =PyType_Ready (&pyrf_mmap_event__type); +if (err <0 ) + goto out; + err =PyType_Ready (&pyrf_mmap2_event__type);if (err <0 ) goto out; err =PyType_Ready (&pyrf_lost_event__type);
@@ -830,6 +974,7 @@ static int pyrf_event__setup_types(void)static PyTypeObject *pyrf_event__type[] = { [PERF_RECORD_MMAP ] = &pyrf_mmap_event__type, + [PERF_RECORD_MMAP2 ] = &pyrf_mmap2_event__type, [PERF_RECORD_LOST ] = &pyrf_lost_event__type, [PERF_RECORD_COMM ] = &pyrf_comm_event__type, [PERF_RECORD_EXIT ] = &pyrf_task_event__type,
@@ -876,6 +1021,9 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evif (copy_size <sizeof (pevent->event))memset ((char *)&pevent->event + copy_size,0 ,sizeof (pevent->event) - copy_size); +if (event->header.type ==PERF_RECORD_MMAP2 ) + pevent->event.mmap2.filename[sizeof (pevent->event.mmap2.filename) -1 ] ='\0' ; +perf_sample__init (&pevent->sample,/*all=*/ true); pevent->al_resolved = false;addr_location__init (&pevent->al);
@@ -3330,6 +3478,9 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_mmap_event__type);PyModule_AddObject (module,"mmap_event" , (PyObject *)&pyrf_mmap_event__type); +Py_INCREF (&pyrf_mmap2_event__type); +PyModule_AddObject (module,"mmap2_event" , (PyObject *)&pyrf_mmap2_event__type); +Py_INCREF (&pyrf_lost_event__type);PyModule_AddObject (module,"lost_event" , (PyObject *)&pyrf_lost_event__type);
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 13/21 perf python: Add callchain support
In reply to Ian RogersImplement pyrf_callchain_node and pyrf_callchain types for lazy iteration over callchain frames. Add callchain property to sample_event.
--- tools/perf/util/python.c | 221 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 217 insertions(+), 4 deletions(-)
tools/perf/util/python.c +217 -4
@@ -87,6 +87,8 @@ struct pyrf_event {struct addr_location al;/** @al_resolved: True when machine__resolve been called. */ bool al_resolved; +/** @callchain: Resolved callchain, eagerly computed if requested. */ +PyObject *callchain;/** @event: The underlying perf_event that may be in a file or ring buffer. */ union perf_event event; };
@@ -124,6 +126,7 @@ static void pyrf_event__delete(struct pyrf_event *pevent) {if (pevent->al_resolved)addr_location__exit (&pevent->al); +Py_XDECREF (pevent->callchain);perf_sample__exit (&pevent->sample);Py_TYPE (pevent)->tp_free ((PyObject *)pevent); }
@@ -785,6 +788,144 @@ static PyObject *pyrf_sample_event__insn(PyObject *self, PyObject *args __maybe_ pevent->sample.insn_len); } +struct pyrf_callchain_node { +PyObject_HEAD + u64 ip; +struct map *map; +struct symbol *sym; +}; + +static void pyrf_callchain_node__delete (struct pyrf_callchain_node *pnode) +{ +map__put (pnode->map); +Py_TYPE (pnode)->tp_free ((PyObject *)pnode); +} + +static PyObject *pyrf_callchain_node__get_ip (struct pyrf_callchain_node *pnode, +void *closure__maybe_unused ) +{ +return PyLong_FromUnsignedLongLong (pnode->ip); +} + +static PyObject *pyrf_callchain_node__get_symbol (struct pyrf_callchain_node *pnode, +void *closure__maybe_unused ) +{ +if (pnode->sym) +return PyUnicode_FromString (pnode->sym->name); +return PyUnicode_FromString ("[unknown]" ); +} + +static PyObject *pyrf_callchain_node__get_dso (struct pyrf_callchain_node *pnode, +void *closure__maybe_unused ) +{ +const char *dsoname ="[unknown]" ; + +if (pnode->map) { +struct dso *dso =map__dso (pnode->map); + +if (dso) { +if (symbol_conf.show_kernel_path &&dso__long_name (dso)) + dsoname =dso__long_name (dso); +else + dsoname =dso__name (dso); + } + } +return PyUnicode_FromString (dsoname); +} + +static PyGetSetDef pyrf_callchain_node__getset[] = { + { .name ="ip" , .get = (getter )pyrf_callchain_node__get_ip, }, + { .name ="symbol" , .get = (getter )pyrf_callchain_node__get_symbol, }, + { .name ="dso" , .get = (getter )pyrf_callchain_node__get_dso, }, + { .name =NULL , }, +}; + +static PyTypeObject pyrf_callchain_node__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.callchain_node" , + .tp_basicsize =sizeof (struct pyrf_callchain_node ), + .tp_dealloc = (destructor )pyrf_callchain_node__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc ="perf callchain node object." , + .tp_getset = pyrf_callchain_node__getset, +}; + +struct pyrf_callchain_frame { +u64 ip; +struct map *map; +struct symbol *sym; +}; + +struct pyrf_callchain { +PyObject_HEAD + struct pyrf_callchain_frame *frames; +u64 nr_frames; +}; + +static void pyrf_callchain__delete (struct pyrf_callchain *pchain) +{ +if (pchain->frames) { +for (u64 i =0 ; i < pchain->nr_frames; i++) +map__put (pchain->frames[i].map); +free (pchain->frames); + } +Py_TYPE (pchain)->tp_free ((PyObject *)pchain); +} + +static Py_ssize_t pyrf_callchain__length (PyObject *obj) +{ +struct pyrf_callchain *pchain = (void *)obj; + +return pchain->nr_frames; +} + +static PyObject *pyrf_callchain__item (PyObject *obj,Py_ssize_t i) +{ +struct pyrf_callchain *pchain = (void *)obj; +struct pyrf_callchain_node *pnode; + +if (i <0 || i >= (Py_ssize_t )pchain->nr_frames) { +PyErr_SetString (PyExc_IndexError,"Index out of range" ); +return NULL ; + } + + pnode = PyObject_New(struct pyrf_callchain_node, &pyrf_callchain_node__type); +if (!pnode) +return NULL ; + + pnode->ip = pchain->frames[i].ip; + pnode->map =map__get (pchain->frames[i].map); + pnode->sym = pchain->frames[i].sym; + +return (PyObject *)pnode; +} + +static PySequenceMethods pyrf_callchain__sequence_methods = { + .sq_length = pyrf_callchain__length, + .sq_item = pyrf_callchain__item, +}; + +static PyTypeObject pyrf_callchain__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.callchain" , + .tp_basicsize =sizeof (struct pyrf_callchain ), + .tp_dealloc = (destructor )pyrf_callchain__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc ="perf callchain object." , + .tp_as_sequence = &pyrf_callchain__sequence_methods, +}; + +static PyObject *pyrf_sample_event__get_callchain (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (!pevent->callchain) + Py_RETURN_NONE; + +Py_INCREF (pevent->callchain); +return pevent->callchain; +} +static PyObject *pyrf_sample_event__getattro (struct pyrf_event *pevent,PyObject *attr_name) {
@@ -799,6 +940,12 @@ pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name) }static PyGetSetDef pyrf_sample_event__getset[] = { + { + .name ="callchain" , + .get = pyrf_sample_event__get_callchain, + .set =NULL , + .doc ="event callchain." , + }, { .name ="raw_buf" , .get = (getter )pyrf_sample_event__get_raw_buf,
@@ -968,6 +1115,12 @@ static int pyrf_event__setup_types(void) err =PyType_Ready (&pyrf_context_switch_event__type);if (err <0 ) goto out; + err =PyType_Ready (&pyrf_callchain_node__type); +if (err <0 ) + goto out; + err =PyType_Ready (&pyrf_callchain__type); +if (err <0 ) + goto out; out:return err; }
@@ -987,13 +1140,19 @@ static PyTypeObject *pyrf_event__type[] = { [PERF_RECORD_SWITCH_CPU_WIDE] = &pyrf_context_switch_event__type, }; -static PyObject *pyrf_event__new (const union perf_event *event,struct evsel *evsel) +static PyObject *pyrf_event__new (const union perf_event *event,struct evsel *evsel, +struct perf_session *session, +struct machine *machine) {struct pyrf_event *pevent; +struct perf_sample *sample;int err;u32 min_size;bool needs_swap; +if (!machine) + machine = session ? &session->machines.host :NULL ; +if (event->header.type >=ARRAY_SIZE (pyrf_event__type) || pyrf_event__type[event->header.type] ==NULL ) { return PyErr_Format(PyExc_TypeError, "Unexpected header type %u",
@@ -1025,6 +1184,7 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *ev pevent->event.mmap2.filename[sizeof (pevent->event.mmap2.filename) -1 ] ='\0' ;perf_sample__init (&pevent->sample,/*all=*/ true); + pevent->callchain =NULL ; pevent->al_resolved = false;addr_location__init (&pevent->al);
@@ -1042,6 +1202,50 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evreturn PyErr_Format (PyExc_OSError,"perf: can't parse sample, err=%d" , err); } + sample = &pevent->sample; +if (machine && sample->callchain) { +struct addr_location al; +struct callchain_cursor *cursor; +u64 i; +struct pyrf_callchain *pchain; + +addr_location__init (&al); +if (machine__resolve (machine, &al, sample) >=0 ) { + cursor =get_tls_callchain_cursor (); +if (thread__resolve_callchain (al.thread, cursor, sample, +NULL ,NULL ,PERF_MAX_STACK_DEPTH ) ==0 ) { +callchain_cursor_commit (cursor); + + pchain = PyObject_New(struct pyrf_callchain, &pyrf_callchain__type); +if (!pchain) { +addr_location__exit (&al); +Py_DECREF (pevent); +return NULL ; + } + pchain->nr_frames = cursor->nr; + pchain->frames =calloc (pchain->nr_frames, +sizeof (*pchain->frames)); +if (!pchain->frames) { +Py_DECREF (pchain); +addr_location__exit (&al); +Py_DECREF (pevent); +return PyErr_NoMemory (); + } +struct callchain_cursor_node *node; + +for (i =0 ; i < pchain->nr_frames; i++) { + node =callchain_cursor_current (cursor); + pchain->frames[i].ip = node->ip; + pchain->frames[i].map = +map__get (node->ms.map); + pchain->frames[i].sym = node->ms.sym; +callchain_cursor_advance (cursor); + } + pevent->callchain = (PyObject *)pchain; + } +addr_location__exit (&al); + } + }return (PyObject *)pevent; }
@@ -2417,7 +2621,7 @@ static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist,perf_mmap__consume (&md->core); Py_RETURN_NONE; } - pyevent =pyrf_event__new (event, evsel); + pyevent =pyrf_event__new (event, evsel,evlist__session (evlist),/*machine=*/ NULL );perf_mmap__consume (&md->core);if (pyevent ==NULL )return PyErr_Occurred () ?NULL :PyErr_NoMemory ();
@@ -3185,10 +3389,10 @@ struct pyrf_session {static int pyrf_session_tool__sample (const struct perf_tool *tool,union perf_event *event,struct perf_sample *sample, -struct machine *machine__maybe_unused ) +struct machine *machine) { struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); -PyObject *pyevent =pyrf_event__new (event, sample->evsel); +PyObject *pyevent =pyrf_event__new (event, sample->evsel, psession->session, machine);PyObject *ret;if (pyevent ==NULL )
@@ -3296,6 +3500,9 @@ static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject } psession->session = session; + symbol_conf.use_callchain = true; + symbol_conf.show_kernel_path = true; + symbol_conf.inline_name = false;if (symbol__init (perf_session__env (session)) <0 ) {PyErr_SetString (PyExc_OSError,"perf: symbol__init failed" ); goto err_out;
@@ -3475,6 +3682,12 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_thread__type);PyModule_AddObject (module,"thread" , (PyObject *)&pyrf_thread__type); +Py_INCREF (&pyrf_callchain__type); +PyModule_AddObject (module,"callchain" , (PyObject *)&pyrf_callchain__type); + +Py_INCREF (&pyrf_callchain_node__type); +PyModule_AddObject (module,"callchain_node" , (PyObject *)&pyrf_callchain_node__type); +Py_INCREF (&pyrf_mmap_event__type);PyModule_AddObject (module,"mmap_event" , (PyObject *)&pyrf_mmap_event__type);
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 14/21 perf python: Extend API for stat events in python.c
In reply to Ian RogersAdd stat information to the session. Add call backs for stat events.
--- tools/perf/util/python.c | 189 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 184 insertions(+), 5 deletions(-)
tools/perf/util/python.c +184 -5
@@ -458,6 +458,77 @@ static PyTypeObject pyrf_lost_event__type = { .tp_repr = (reprfunc)pyrf_lost_event__repr, }; +static const char pyrf_stat_event__doc[] =PyDoc_STR ("perf stat event object." ); + +static PyMemberDef pyrf_stat_event__members[] = { + sample_members +member_def (perf_event_header, type,T_UINT ,"event type" ), +member_def (perf_record_stat, id,T_ULONGLONG ,"event id" ), +member_def (perf_record_stat, cpu,T_UINT ,"event cpu" ), +member_def (perf_record_stat, thread,T_UINT ,"event thread" ), +member_def (perf_record_stat, val,T_ULONGLONG ,"counter value" ), +member_def (perf_record_stat, ena,T_ULONGLONG ,"enabled time" ), +member_def (perf_record_stat, run,T_ULONGLONG ,"running time" ), + { .name =NULL , }, +}; + +static PyObject *pyrf_stat_event__repr (const struct pyrf_event *pevent) +{ +return PyUnicode_FromFormat ( +"{ type: stat, id: %llu, cpu: %u, thread: %u, val: %llu, ena: %llu, run: %llu }" , + pevent->event.stat.id, + pevent->event.stat.cpu, + pevent->event.stat.thread, + pevent->event.stat.val, + pevent->event.stat.ena, + pevent->event.stat.run); +} + +static PyTypeObject pyrf_stat_event__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.stat_event" , + .tp_basicsize =sizeof (struct pyrf_event ), + .tp_new = PyType_GenericNew, + .tp_dealloc = (destructor )pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_stat_event__doc, + .tp_members = pyrf_stat_event__members, + .tp_getset = pyrf_event__getset, + .tp_repr = (reprfunc )pyrf_stat_event__repr, +}; + +static const char pyrf_stat_round_event__doc[] =PyDoc_STR ("perf stat round event object." ); + +static PyMemberDef pyrf_stat_round_event__members[] = { + sample_members +member_def (perf_event_header, type,T_UINT ,"event type" ), + { .name ="stat_round_type" , .type =T_ULONGLONG , + .offset = offsetof(struct pyrf_event , event) + offsetof(struct perf_record_stat_round , type), + .doc ="round type" }, +member_def (perf_record_stat_round, time,T_ULONGLONG ,"round time" ), + { .name =NULL , }, +}; + +static PyObject *pyrf_stat_round_event__repr (const struct pyrf_event *pevent) +{ +return PyUnicode_FromFormat ("{ type: stat_round, type: %llu, time: %llu }" , + pevent->event.stat_round.type, + pevent->event.stat_round.time); +} + +static PyTypeObject pyrf_stat_round_event__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.stat_round_event" , + .tp_basicsize =sizeof (struct pyrf_event ), + .tp_new = PyType_GenericNew, + .tp_dealloc = (destructor )pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_stat_round_event__doc, + .tp_members = pyrf_stat_round_event__members, + .tp_getset = pyrf_event__getset, + .tp_repr = (reprfunc )pyrf_stat_round_event__repr, +}; +static const char pyrf_read_event__doc[] =PyDoc_STR ("perf read event object." );static PyMemberDef pyrf_read_event__members[] = {
@@ -1113,6 +1184,12 @@ static int pyrf_event__setup_types(void)if (err <0 ) goto out; err =PyType_Ready (&pyrf_context_switch_event__type); +if (err <0 ) + goto out; + err =PyType_Ready (&pyrf_stat_event__type); +if (err <0 ) + goto out; + err =PyType_Ready (&pyrf_stat_round_event__type);if (err <0 ) goto out; err =PyType_Ready (&pyrf_callchain_node__type);
@@ -1138,6 +1215,8 @@ static PyTypeObject *pyrf_event__type[] = { [PERF_RECORD_SAMPLE] = &pyrf_sample_event__type, [PERF_RECORD_SWITCH] = &pyrf_context_switch_event__type, [PERF_RECORD_SWITCH_CPU_WIDE] = &pyrf_context_switch_event__type, + [PERF_RECORD_STAT] = &pyrf_stat_event__type, + [PERF_RECORD_STAT_ROUND] = &pyrf_stat_round_event__type, }; static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evsel,
@@ -2132,7 +2211,45 @@ static PyObject *pyrf_evsel__get_attr_wakeup_events(PyObject *self, void *closurreturn PyLong_FromUnsignedLong (pevsel->evsel->core.attr.wakeup_events); } +static PyObject *pyrf_evsel__get_ids (struct pyrf_evsel *pevsel,void *closure__maybe_unused ) +{ +struct evsel *evsel; +PyObject *list; + +CHECK_INITIALIZED (pevsel->evsel,"evsel" ); + + evsel = pevsel->evsel; + list =PyList_New (0 ); + +if (!list) +return NULL ; + +for (u32 i =0 ; i < evsel->core.ids; i++) { +PyObject *id =PyLong_FromUnsignedLongLong (evsel->core.id[i]); +int ret; + +if (!id) { +Py_DECREF (list); +return NULL ; + } + ret =PyList_Append (list, id); +Py_DECREF (id); +if (ret <0 ) { +Py_DECREF (list); +return NULL ; + } + } + +return list; +} +static PyGetSetDef pyrf_evsel__getset[] = { + { + .name ="ids" , + .get = (getter )pyrf_evsel__get_ids, + .set =NULL , + .doc ="event IDs." , + }, { .name ="tracking" , .get = pyrf_evsel__get_tracking,
@@ -2985,6 +3102,8 @@ static const struct perf_constant perf__constants[] = {PERF_CONST (RECORD_LOST_SAMPLES ),PERF_CONST (RECORD_SWITCH ),PERF_CONST (RECORD_SWITCH_CPU_WIDE ), +PERF_CONST (RECORD_STAT ), +PERF_CONST (RECORD_STAT_ROUND ),PERF_CONST (RECORD_MISC_SWITCH_OUT ), { .name = NULL, },
@@ -3384,6 +3503,7 @@ struct pyrf_session {struct perf_tool tool;struct pyrf_data *pdata;PyObject *sample; +PyObject *stat; }; static int pyrf_session_tool__sample(const struct perf_tool *tool,
@@ -3408,6 +3528,52 @@ static int pyrf_session_tool__sample(const struct perf_tool *tool,return 0 ; } +static int pyrf_session_tool__stat (const struct perf_tool *tool, +struct perf_session *session, +union perf_event *event) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); +struct evsel *evsel =evlist__id2evsel (session->evlist, event->stat.id); +PyObject *pyevent =pyrf_event__new (event,/*evsel=*/ NULL , psession->session, +/*machine=*/ NULL ); +const char *name = evsel ?evsel__name (evsel) :"unknown" ; +PyObject *ret; + +if (pyevent ==NULL ) +return -ENOMEM ; + + ret =PyObject_CallFunction (psession->stat,"Oz" , pyevent, name); +if (!ret) { +Py_DECREF (pyevent); +return -1 ; + } +Py_DECREF (ret); +Py_DECREF (pyevent); +return 0 ; +} + +static int pyrf_session_tool__stat_round (const struct perf_tool *tool, +struct perf_session *session__maybe_unused , +union perf_event *event) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); +PyObject *pyevent =pyrf_event__new (event,/*evsel=*/ NULL , psession->session, +/*machine=*/ NULL ); +PyObject *ret; + +if (pyevent ==NULL ) +return -ENOMEM ; + + ret =PyObject_CallFunction (psession->stat,"Oz" , pyevent,NULL ); +if (!ret) { +Py_DECREF (pyevent); +return -1 ; + } +Py_DECREF (ret); +Py_DECREF (pyevent); +return 0 ; +} +static PyObject *pyrf_session__find_thread (struct pyrf_session *psession,PyObject *args) {struct machine *machine;
@@ -3441,13 +3607,13 @@ static PyObject *pyrf_session__find_thread(struct pyrf_session *psession, PyObjestatic PyObject *pyrf_session__new (PyTypeObject *type,PyObject *args,PyObject *kwargs) {struct pyrf_data *pdata; -PyObject *sample =NULL ; -static char *kwlist[] = {"data" ,"sample" ,NULL }; +PyObject *sample =NULL , *stat =NULL ; +static char *kwlist[] = {"data" ,"sample" ,"stat" ,NULL };struct pyrf_session *psession;struct perf_session *session; -if (!PyArg_ParseTupleAndKeywords (args, kwargs,"O!|O" , kwlist, &pyrf_data__type, &pdata, - &sample)) +if (!PyArg_ParseTupleAndKeywords (args, kwargs,"O!|OO" , kwlist, &pyrf_data__type, &pdata, + &sample, &stat))return NULL ; psession = PyObject_New(struct pyrf_session, type);
@@ -3456,6 +3622,7 @@ static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject psession->session =NULL ; psession->sample =NULL ; + psession->stat =NULL ; psession->pdata =NULL ;Py_INCREF (pdata);
@@ -3478,8 +3645,13 @@ static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject }while (0 )ADD_TOOL (sample); +ADD_TOOL (stat);#undef ADD_TOOL +if (stat) + psession->tool.stat_round = pyrf_session_tool__stat_round; + + psession->tool.comm = perf_event__process_comm; psession->tool.mmap = perf_event__process_mmap; psession->tool.mmap2 = perf_event__process_mmap2;
@@ -3492,7 +3664,7 @@ static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject psession->tool.build_id = perf_event__process_build_id; psession->tool.attr = perf_event__process_attr; psession->tool.feature = perf_event__process_feature; - psession->tool.stat = perf_event__process_stat_event; + session =perf_session__new (&pdata->data, &psession->tool);if (IS_ERR (session)) {PyErr_Format (PyExc_IOError,"failed to create session: %ld" ,PTR_ERR (session));
@@ -3521,6 +3693,7 @@ static void pyrf_session__delete(struct pyrf_session *psession)perf_session__delete (psession->session);Py_XDECREF (psession->pdata);Py_XDECREF (psession->sample); +Py_XDECREF (psession->stat);Py_TYPE (psession)->tp_free ((PyObject *)psession); }
@@ -3718,6 +3891,12 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_context_switch_event__type);PyModule_AddObject (module,"switch_event" , (PyObject *)&pyrf_context_switch_event__type); +Py_INCREF (&pyrf_stat_event__type); +PyModule_AddObject (module,"stat_event" , (PyObject *)&pyrf_stat_event__type); + +Py_INCREF (&pyrf_stat_round_event__type); +PyModule_AddObject (module,"stat_round_event" , (PyObject *)&pyrf_stat_round_event__type); +Py_INCREF (&pyrf_thread_map__type);PyModule_AddObject (module,"thread_map" , (PyObject *)&pyrf_thread_map__type);
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 15/21 perf python: Expose brstack in sample event
In reply to Ian RogersImplement pyrf_branch_entry and pyrf_branch_stack for lazy iteration over branch stack entries.
--- tools/perf/util/python.c | 198 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+)
tools/perf/util/python.c +198 -0
@@ -22,6 +22,7 @@#include "debug.h" #include "dso.h" #include "event.h" +#include "branch.h" #include "evlist.h" #include "evsel.h" #include "expr.h"
@@ -89,6 +90,8 @@ struct pyrf_event {bool al_resolved;/** @callchain: Resolved callchain, eagerly computed if requested. */ PyObject *callchain; +/** @brstack: Resolved branch stack, eagerly computed if requested. */ +PyObject *brstack;/** @event: The underlying perf_event that may be in a file or ring buffer. */ union perf_event event; };
@@ -127,6 +130,7 @@ static void pyrf_event__delete(struct pyrf_event *pevent)if (pevent->al_resolved)addr_location__exit (&pevent->al);Py_XDECREF (pevent->callchain); +Py_XDECREF (pevent->brstack);perf_sample__exit (&pevent->sample);Py_TYPE (pevent)->tp_free ((PyObject *)pevent); }
@@ -997,6 +1001,154 @@ static PyObject *pyrf_sample_event__get_callchain(PyObject *self, void *closurereturn pevent->callchain; } +struct pyrf_branch_entry { +PyObject_HEAD + u64 from; +u64 to; +struct branch_flags flags; +}; + +static void pyrf_branch_entry__delete (struct pyrf_branch_entry *pentry) +{ +Py_TYPE (pentry)->tp_free ((PyObject *)pentry); +} + +static PyObject *pyrf_branch_entry__get_from (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyLong_FromUnsignedLongLong (pentry->from); +} + +static PyObject *pyrf_branch_entry__get_to (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyLong_FromUnsignedLongLong (pentry->to); +} + +static PyObject *pyrf_branch_entry__get_mispred (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyBool_FromLong (pentry->flags.mispred); +} + +static PyObject *pyrf_branch_entry__get_predicted (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyBool_FromLong (pentry->flags.predicted); +} + +static PyObject *pyrf_branch_entry__get_in_tx (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyBool_FromLong (pentry->flags.in_tx); +} + +static PyObject *pyrf_branch_entry__get_abort (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyBool_FromLong (pentry->flags.abort); +} + +static PyObject *pyrf_branch_entry__get_cycles (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyLong_FromUnsignedLongLong (pentry->flags.cycles); +} + +static PyObject *pyrf_branch_entry__get_type (struct pyrf_branch_entry *pentry, +void *closure__maybe_unused ) +{ +return PyLong_FromUnsignedLongLong ((unsigned long long )pentry->flags.type); +} + +static PyGetSetDef pyrf_branch_entry__getset[] = { + { .name ="from_ip" , .get = (getter )pyrf_branch_entry__get_from, }, + { .name ="to_ip" , .get = (getter )pyrf_branch_entry__get_to, }, + { .name ="mispred" , .get = (getter )pyrf_branch_entry__get_mispred, }, + { .name ="predicted" , .get = (getter )pyrf_branch_entry__get_predicted, }, + { .name ="in_tx" , .get = (getter )pyrf_branch_entry__get_in_tx, }, + { .name ="abort" , .get = (getter )pyrf_branch_entry__get_abort, }, + { .name ="cycles" , .get = (getter )pyrf_branch_entry__get_cycles, }, + { .name ="type" , .get = (getter )pyrf_branch_entry__get_type, }, + { .name =NULL , }, +}; + +static PyTypeObject pyrf_branch_entry__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.branch_entry" , + .tp_basicsize =sizeof (struct pyrf_branch_entry ), + .tp_dealloc = (destructor )pyrf_branch_entry__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc ="perf branch entry object." , + .tp_getset = pyrf_branch_entry__getset, +}; + +struct pyrf_branch_stack { +PyObject_HEAD + struct branch_entry *entries; +u64 nr; +}; + +static void pyrf_branch_stack__delete (struct pyrf_branch_stack *pstack) +{ +free (pstack->entries); +Py_TYPE (pstack)->tp_free ((PyObject *)pstack); +} + +static Py_ssize_t pyrf_branch_stack__length (PyObject *obj) +{ +struct pyrf_branch_stack *pstack = (void *)obj; + +return pstack->nr; +} + +static PyObject *pyrf_branch_stack__item (PyObject *obj,Py_ssize_t i) +{ +struct pyrf_branch_stack *pstack = (void *)obj; +struct pyrf_branch_entry *pentry; + +if (i <0 || i >= (Py_ssize_t )pstack->nr) { +PyErr_SetString (PyExc_IndexError,"Index out of range" ); +return NULL ; + } + + pentry = PyObject_New(struct pyrf_branch_entry, &pyrf_branch_entry__type); +if (!pentry) +return NULL ; + + pentry->from = pstack->entries[i].from; + pentry->to = pstack->entries[i].to; + pentry->flags = pstack->entries[i].flags; + +return (PyObject *)pentry; +} + +static PySequenceMethods pyrf_branch_stack__sequence_methods = { + .sq_length = pyrf_branch_stack__length, + .sq_item = pyrf_branch_stack__item, +}; + +static PyTypeObject pyrf_branch_stack__type = { +PyVarObject_HEAD_INIT (NULL ,0 ) + .tp_name ="perf.branch_stack" , + .tp_basicsize =sizeof (struct pyrf_branch_stack ), + .tp_dealloc = (destructor )pyrf_branch_stack__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc ="perf branch stack object." , + .tp_as_sequence = &pyrf_branch_stack__sequence_methods, +}; + +static PyObject *pyrf_sample_event__get_brstack (PyObject *self,void *closure__maybe_unused ) +{ +struct pyrf_event *pevent = (void *)self; + +if (!pevent->brstack) + Py_RETURN_NONE; + +Py_INCREF (pevent->brstack); +return pevent->brstack; +} +static PyObject *pyrf_sample_event__getattro (struct pyrf_event *pevent,PyObject *attr_name) {
@@ -1017,6 +1169,12 @@ static PyGetSetDef pyrf_sample_event__getset[] = { .set = NULL, .doc = "event callchain.", }, + { + .name = "brstack", + .get = pyrf_sample_event__get_brstack, + .set = NULL, + .doc = "event branch stack.", + }, { .name = "raw_buf", .get = (getter)pyrf_sample_event__get_raw_buf,
@@ -1198,6 +1356,12 @@ static int pyrf_event__setup_types(void) err =PyType_Ready (&pyrf_callchain__type);if (err <0 ) goto out; + err =PyType_Ready (&pyrf_branch_entry__type); +if (err <0 ) + goto out; + err =PyType_Ready (&pyrf_branch_stack__type); +if (err <0 ) + goto out; out:return err; }
@@ -1264,6 +1428,7 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evperf_sample__init (&pevent->sample,/*all=*/ true); pevent->callchain =NULL ; + pevent->brstack =NULL ; pevent->al_resolved = false;addr_location__init (&pevent->al);
@@ -1325,6 +1490,27 @@ static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evaddr_location__exit (&al); } } +if (sample->branch_stack) { +struct branch_stack *bs = sample->branch_stack; +struct branch_entry *entries =perf_sample__branch_entries (sample); +struct pyrf_branch_stack *pstack; + + pstack = PyObject_New(struct pyrf_branch_stack, &pyrf_branch_stack__type); +if (!pstack) { +Py_DECREF (pevent); +return NULL ; + } + pstack->nr = bs->nr; + pstack->entries =calloc (bs->nr,sizeof (struct branch_entry )); +if (!pstack->entries) { +Py_DECREF (pstack); +Py_DECREF (pevent); +return PyErr_NoMemory (); + } +memcpy (pstack->entries, entries, + bs->nr *sizeof (struct branch_entry )); + pevent->brstack = (PyObject *)pstack; + }return (PyObject *)pevent; }
@@ -3912,6 +4098,18 @@ PyMODINIT_FUNC PyInit_perf(void)Py_INCREF (&pyrf_session__type);PyModule_AddObject (module,"session" , (PyObject *)&pyrf_session__type); +Py_INCREF (&pyrf_branch_entry__type); +if (PyModule_AddObject (module,"branch_entry" , (PyObject *)&pyrf_branch_entry__type) <0 ) { +Py_DECREF (&pyrf_branch_entry__type); + goto error; + } + +Py_INCREF (&pyrf_branch_stack__type); +if (PyModule_AddObject (module,"branch_stack" , (PyObject *)&pyrf_branch_stack__type) <0 ) { +Py_DECREF (&pyrf_branch_stack__type); + goto error; + } + dict =PyModule_GetDict (module);if (dict ==NULL ) goto error;
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 16/21 perf python: Add syscall name/id to convert syscall number and name
In reply to Ian RogersUse perf's syscalltbl support to convert syscall number to name assuming the number is for the host machine. This avoids python libaudit support as tools/perf/scripts/python/syscall-counts.py requires.
--- tools/perf/util/Build | 1 - tools/perf/util/python.c | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-)
tools/perf/util/Build +0 -1
@@ -75,7 +75,6 @@ perf-util-y += sample.o perf-util-y += sample-raw.o perf-util-y += s390-sample-raw.o perf-util-y += amd-sample-raw.o - perf-util-y += ordered-events.o perf-util-y += namespaces.o perf-util-y += comm.o
tools/perf/util/python.c +48 -0
72 lines not on this page · raw message
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 16/21] perf python: Add syscall name/id to convert syscall number and name
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Medium] The commit introduces `syscall_name` and `syscall_id` to the `perf` Python extension explicitly…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The commit introduces syscall_name and syscall_id to the perf Python extension explicitly to avoid relying on python-audit for tools/perf/scripts/python/syscall-counts.py as stated in the commit message. However, the patch series fails to actually update syscall-counts.py or its utility libraries (Util.py) to use these new functions, leaving an incomplete implementation and a mismatch between the commit message and the actual code changes. --
commit 5f14858d1e943fba0e189e558464ae637b5fb965
Author: Ian Rogers <[email protected]>
perf python: Add syscall name/id to convert syscall number and name
This commit adds syscall_name and syscall_id methods to the perf Python extension to map between syscall numbers and names. The commit message states this is intended to replace the libaudit dependency in syscall-counts.py.
Quoted 3 lines
tools/perf/util/python.c +0
-0
[ ... ]
tools/perf/util/python.c static PyMethodDef perf__methods[] = { · line 4031
Quoted 7 lines
tools/perf/util/python.c +3
-0@@ -3990,6 +4026,18 @@ static PyMethodDef perf__methods[] = { .ml_flags = METH_NOARGS, .ml_doc =PyDoc_STR ("Returns a sequence of pmus." ) }, + { + .ml_name = "syscall_name", + .ml_meth = (PyCFunction) pyrf__syscall_name,
[Severity: Medium]
The commit message states that adding these methods avoids the python libaudit support required by tools/perf/scripts/python/syscall-counts.py.
However, the patch doesn't appear to update syscall-counts.py or its helper modules to actually use these new methods. Should the script update be included in this patch, or is it planned for a later patch in the series?
Quoted 11 lines
tools/perf/util/python.c +9
-0+ .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("Turns a syscall number to a string.") + }, + { + .ml_name = "syscall_id", + .ml_meth = (PyCFunction) pyrf__syscall_id, + .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("Turns a syscall name to a number.") + }, { .ml_name = NULL, } };
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=16
Ian Rogers
reviewerPATCH v21 17/21 perf python: Add config file access
In reply to Ian RogersAdd perf.config_get(name) to expose the perf configuration system.
--- tools/perf/util/python.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
tools/perf/util/python.c +20 -0
@@ -17,6 +17,7 @@#include "build-id.h" #include "callchain.h" #include "comm.h" +#include "config.h" #include "counts.h" #include "data.h" #include "debug.h"
@@ -3993,7 +3994,26 @@ static PyObject *pyrf__syscall_id(PyObject *self, PyObject *args, PyObject *kwarreturn PyLong_FromLong (id); } +static PyObject *pyrf__config_get (PyObject *self,PyObject *args) +{ +const char *config_name, *val; + +if (!PyArg_ParseTuple (args,"s" , &config_name)) +return NULL ; + + val =perf_config_get (config_name); +if (!val) + Py_RETURN_NONE; +return PyUnicode_FromString (val); +} +static PyMethodDef perf__methods[] = { + { + .ml_name ="config_get" , + .ml_meth = (PyCFunction ) pyrf__config_get, + .ml_flags =METH_VARARGS , + .ml_doc =PyDoc_STR ("Get a perf config value." ) + }, { .ml_name ="metrics" , .ml_meth = (PyCFunction ) pyrf__metrics,
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v21 18/21 perf python: Handle Py_None for thread and cpu maps
In reply to Ian RogersThe python stubs allow passing None for threads and cpus to the perf.parse_events() and perf.parse_metrics() bindings. However, PyArg_ParseTuple parses None into a Py_None object, which is not a NULL pointer. Because the C code lacked an explicit check for Py_None, it would cast Py_None to a pyrf_thread_map/pyrf_cpu_map struct pointer and dereference it, causing a memory corruption crash.
Fix this pre-existing issue by explicitly checking for Py_None alongside NULL in pyrf__parse_events, pyrf__parse_metrics, and pyrf_evsel__open.
--- tools/perf/util/python.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-)
tools/perf/util/python.c +10 -6
@@ -2091,10 +2091,10 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, &pcpus, &pthreads, &group, &inherit))return NULL ; -if (pthreads !=NULL ) +if (pthreads !=NULL && pthreads != Py_None) threads = ((struct pyrf_thread_map *)pthreads)->threads; -if (pcpus !=NULL ) +if (pcpus !=NULL && pcpus != Py_None) cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; evsel->core.attr.inherit = inherit;
@@ -3341,8 +3341,10 @@ static PyObject *pyrf__parse_events(PyObject *self, PyObject *args)return NULL ; } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads :NULL ; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus :NULL ; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads :NULL ; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus :NULL ;parse_events_error__init (&err);perf_evlist__set_maps (evlist__core (evlist), cpus, threads);
@@ -3375,8 +3377,10 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args)return NULL ; } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads :NULL ; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus :NULL ; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads :NULL ; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus :NULL ;perf_evlist__set_maps (evlist__core (evlist), cpus, threads); ret = metricgroup__parse_groups(evlist, pmu ?: "all", input,
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 18/21] perf python: Handle Py_None for thread and cpu maps
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
Pre-existing issues:
- [High] Missing Python object type check before pointer cast in pyrf_evsel__open leads to type confusion and potential memory corruption. --
commit 3a81d3b48b518ddfa19a43ac15366264b0a3588b
Author: Ian Rogers <[email protected]>
perf python: Handle Py_None for thread and cpu maps
This commit fixes a memory corruption issue where passing None for threads and cpus to perf bindings results in Py_None being incorrectly cast and dereferenced. It adds explicit checks for Py_None alongside NULL in several Python binding functions.
Quoted 3 lines
tools/perf/util/python.c +0
-0
[ ... ]
tools/perf/util/python.c static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, · line 2098
Quoted 11 lines
tools/perf/util/python.c +2
-2@@ -2091,10 +2091,10 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, &pcpus, &pthreads, &group, &inherit))return NULL ; -if (pthreads !=NULL ) +if (pthreads !=NULL && pthreads != Py_None) threads = ((struct pyrf_thread_map *)pthreads)->threads; -if (pcpus !=NULL ) +if (pcpus !=NULL && pcpus != Py_None) cpus = ((struct pyrf_cpu_map *)pcpus)->cpus;
[Severity: High]
This is a pre-existing issue, but is it possible for a user to pass an arbitrary Python object (like an integer or string) instead of a thread or cpu map here?
It looks like pyrf_evsel__open() uses the "|OOii" format specifier to parse generic PyObject pointers, but it does not verify the object types before casting the pointers to struct pyrf_thread_map and struct pyrf_cpu_map.
While other functions like pyrf__parse_events() and pyrf__parse_metrics() use PyObject_TypeCheck() to validate these arguments, pyrf_evsel__open() seems to be missing this validation. If an invalid object type is provided, could casting and dereferencing it cause memory corruption?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=18
Ian Rogers
reviewerPATCH v21 19/21 perf python: Add type checking for parse_events/parse_metrics
In reply to Ian RogersThe threads and cpus parameters in parse_events and parse_metrics are parsed with the 'O' format specifier but blindly casted in the C extension. If a user passes an invalid object type, this leads to memory corruption when dereferencing the expected struct.
Add runtime PyObject_TypeCheck validations in python.c to safely raise a TypeError if an invalid object is passed.
--- tools/perf/util/python.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)
tools/perf/util/python.c +28 -0
@@ -3341,6 +3341,20 @@ static PyObject *pyrf__parse_events(PyObject *self, PyObject *args)return NULL ; } +if (pthreads && pthreads != Py_None && + !PyObject_TypeCheck (pthreads, &pyrf_thread_map__type)) { +PyErr_SetString (PyExc_TypeError,"threads must be a perf.thread_map or None" ); +evlist__put (evlist); +return NULL ; + } + +if (pcpus && pcpus != Py_None && + !PyObject_TypeCheck (pcpus, &pyrf_cpu_map__type)) { +PyErr_SetString (PyExc_TypeError,"cpus must be a perf.cpu_map or None" ); +evlist__put (evlist); +return NULL ; + } + threads = (pthreads && pthreads != Py_None) ? ((struct pyrf_thread_map *)pthreads)->threads :NULL ; cpus = (pcpus && pcpus != Py_None) ?
@@ -3377,6 +3391,20 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args)return NULL ; } +if (pthreads && pthreads != Py_None && + !PyObject_TypeCheck (pthreads, &pyrf_thread_map__type)) { +PyErr_SetString (PyExc_TypeError,"threads must be a perf.thread_map or None" ); +evlist__put (evlist); +return NULL ; + } + +if (pcpus && pcpus != Py_None && + !PyObject_TypeCheck (pcpus, &pyrf_cpu_map__type)) { +PyErr_SetString (PyExc_TypeError,"cpus must be a perf.cpu_map or None" ); +evlist__put (evlist); +return NULL ; + } + threads = (pthreads && pthreads != Py_None) ? ((struct pyrf_thread_map *)pthreads)->threads :NULL ; cpus = (pcpus && pcpus != Py_None) ?
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 19/21] perf python: Add type checking for parse_events/parse_metrics
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
Pre-existing issues:
- [High] Missing type validation in pyrf_evsel__open allows invalid Python objects to be blindly casted to perf.cpu_map or perf.thread_map, leading to memory corruption. --
commit ab2b8da876dfec8d6867d13d81bf282eb7c51dcd
Author: Ian Rogers <[email protected]>
perf python: Add type checking for parse_events/parse_metrics
The threads and cpus parameters in parse_events and parse_metrics are parsed with the O format specifier but blindly casted in the C extension. This commit adds runtime PyObject_TypeCheck validations to safely raise a TypeError if an invalid object is passed.
Quoted 11 lines
tools/perf/util/python.c +3
-0@@ -3341,6 +3341,20 @@ static PyObject *pyrf__parse_events(PyObject *self, PyObject *args)return NULL ; } +if (pthreads && pthreads != Py_None && + !PyObject_TypeCheck (pthreads, &pyrf_thread_map__type)) { +PyErr_SetString (PyExc_TypeError,"threads must be a perf.thread_map or None" );
tools/perf/util/python.c static PyObject *pyrf__parse_events(PyObject *self, PyObject *args) · line 3356
Quoted 10 lines
tools/perf/util/python.c +10
-0+evlist__put (evlist); +return NULL ; + } + +if (pcpus && pcpus != Py_None && + !PyObject_TypeCheck (pcpus, &pyrf_cpu_map__type)) { +PyErr_SetString (PyExc_TypeError,"cpus must be a perf.cpu_map or None" ); +evlist__put (evlist); +return NULL ; + }
[Severity: High]
This is a pre-existing issue, but does pyrf_evsel__open() need these same type checks?
Looking at tools/perf/util/python.c in pyrf_evsel__open(), it parses the pcpus and pthreads objects using the |OOii format specifier, but skips PyObject_TypeCheck:
if (pthreads != NULL && pthreads != Py_None) threads = ((struct pyrf_thread_map *)pthreads)->threads; if (pcpus != NULL && pcpus != Py_None) cpus = ((struct pyrf_cpu_map *)pcpus)->cpus;
Could an invalid Python object passed to pyrf_evsel__open() lead to the same memory corruption when these are blindly cast and dereferenced?
[ ... ]
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=19
Ian Rogers
reviewerPATCH v21 20/21 perf python: Add perf.pyi stubs file
In reply to Ian RogersAdd Python type stubs for the perf module to improve IDE support and static analysis. Includes docstrings for classes, methods, and constants derived from C source and JSON definitions.
--- tools/perf/Makefile.perf | 7 +- tools/perf/python/perf.pyi | 672 +++++++++++++++++++++++++++++++++++++ tools/perf/util/setup.py | 5 + 3 files changed, 682 insertions(+), 2 deletions(-) create mode 100644 tools/perf/python/perf.pyi
tools/perf/Makefile.perf +5 -2
@@ -517,12 +517,15 @@ all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) # Create python binding output directory if not already present $(shell [ -d '$(OUTPUT)python' ] || mkdir -p '$(OUTPUT)python') -$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py $(PERFLIBS_PY) +$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py python/perf.pyi $(PERFLIBS_PY) $(QUIET_GEN)LDSHARED="$(CC) -pthread -shared" \ CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS) $(LIBS_PY)' \ $(PYTHON_WORD) util/setup.py \ $(python_setup_quiet) build_ext; \ - cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/ + cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/; \ + if [ "$(abspath python/perf.pyi)" != "$(abspath $(OUTPUT)python/perf.pyi)" ]; then \ + cp python/perf.pyi $(OUTPUT)python/; \ + fi python_perf_target: @echo "Target is: $(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX)"
tools/perf/python/perf.pyi new file +672 -0
672 lines not on this page · raw message
tools/perf/util/setup.py +5 -0
@@ -2,6 +2,7 @@ from os import getenv, path from subprocess import Popen, PIPE from re import sub import shlex +import shutil cc = getenv("CC") assert cc, "Environment variable CC not set"
@@ -73,6 +74,10 @@ class install_lib(_install_lib): _install_lib.finalize_options(self) self.build_dir = build_lib + def run(self): + _install_lib.run(self) + shutil.copy2(f'{src_perf}/python/perf.pyi', self.install_dir) + # switch off several checks (need to be at the end of cflags list) extra_cflags = ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ]
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
[email protected]
AI reviewRe: [PATCH v21 20/21] perf python: Add perf.pyi stubs file
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Medium] The `metrics()` function type stub incorrectly specifies the return type as `List[Dict[str…
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The metrics() function type stub incorrectly specifies the return type as List[Dict[str, str]]. --
commit 15488fc0400fa58c2b16fb87ba64c62006a2314e
Author: Ian Rogers <[email protected]>
perf python: Add perf.pyi stubs file
This commit adds Python type stubs for the perf module to improve IDE support and static analysis. The stubs include docstrings for classes, methods, and constants derived from C sources and JSON definitions.
Quoted 8 lines
tools/perf/python/perf.pyi new file +2
-0@@ -0,0 +1,672 @@ +"""Type stubs for the perf Python module.""" +from typing import Callable, Dict, List, Optional, Any, Iterator
[ ... ]
Quoted 1 line
+def
metrics()-> List[Dict[str, str]]:
[Severity: Medium]
In the C implementation, specifically pyrf__metrics_cb() and pyrf__metrics_groups() in tools/perf/util/python.c, the returned dictionary contains a MetricGroup key which is populated with a Python list of strings, not a single string.
Does this type stub incorrectly restrict the dictionary values to strings only? Static analysis tools might flag valid list operations on MetricGroup values as type errors if they expect every value to strictly be a string.
Quoted 6 lines
tools/perf/python/perf.pyi new file +6
-0+ """Get a list of available metrics. + + Returns: + A list of dictionaries, each describing a metric. + """ + ...
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=20
Ian Rogers
reviewerPATCH v21 21/21 perf python: Add LiveSession helper
In reply to Ian RogersAdd LiveSession class in tools/perf/python/perf_live.py to support live event collection using perf.evlist and perf.parse_events, avoiding the need to fork a separate perf record process.
--- tools/perf/python/perf_live.py | 59 ++++++++++++++++++++++ tools/perf/tests/shell/lib/setup_python.sh | 13 +++++ 2 files changed, 72 insertions(+) create mode 100755 tools/perf/python/perf_live.py
tools/perf/python/perf_live.py new file +59 -0
@@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Live event session helper using perf.evlist. + +This module provides a LiveSession class that allows running a callback +for each event collected live from the system, similar to perf.session +but without requiring a perf.data file. +""" + +import perf + + +class LiveSession: + """Represents a live event collection session.""" + + def __init__(self, event_string: str, sample_callback): + self.event_string = event_string + self.sample_callback = sample_callback + # Create a cpu map for all online CPUs + self.cpus = perf.cpu_map() + # Parse events and set maps + self.evlist = perf.parse_events(self.event_string, self.cpus) + self.evlist.config() + + def run(self): + """Run the live session.""" + try: + self.evlist.open() + self.evlist.mmap() + self.evlist.enable() + + while True: + # Poll for events with 100ms timeout + try: + self.evlist.poll(100) + except InterruptedError: + continue + for cpu in self.cpus: + for _ in range(1000): # Limit to 1000 events per CPU per poll to prevent starvation + try: + event = self.evlist.read_on_cpu(cpu) + except TypeError as e: + if "Unknown CPU" in str(e): + # CPU might be unmapped or offline, wait for mmap event + break + if "Unexpected header type" in str(e): + # Ignore valid but unsupported event types + continue + raise + + if event is None: + break + + if event.type == perf.RECORD_SAMPLE: + self.sample_callback(event) + except KeyboardInterrupt: + pass + finally: + self.evlist.close()
tools/perf/tests/shell/lib/setup_python.sh +13 -0
@@ -14,3 +14,16 @@ then echo Skipping test, python not detected please set environment variable PYTHON. exit 2 fi + +# Set PYTHONPATH to find the in-tree built perf.so first, avoiding system-wide perf.so +if [ -n "$PERF_EXEC_PATH" ] && [ -d "$PERF_EXEC_PATH/python" ]; then + PYTHONPATH_DIR="$PERF_EXEC_PATH/python" +elif [ -d "$(dirname "$0")/../../python" ]; then + PYTHONPATH_DIR="$(dirname "$0")/../../python" +elif [ -d "$(dirname "$0")/../python" ]; then + PYTHONPATH_DIR="$(dirname "$0")/../python" +fi + +if [ -n "$PYTHONPATH_DIR" ]; then + export PYTHONPATH="$PYTHONPATH_DIR${PYTHONPATH:+:$PYTHONPATH}" +fi
Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerRe: [PATCH v21 00/21] Add additional python API support
In reply to Ian RogersSo I think this is the point where I stop trying to address every Sashiko point and hope for human review to land these patches. Here is the Sashiko feedback:
https://sashiko.dev/#/patchset/20260616011543.4037138-1-irogers%40google.com
To summarize: all raised issues are either already addressed or align perfectly with the objectives planned for Phase 2.
Specifically:
- **Build and Reference Counting Fixes:** The missing evlist__nr_entries() accessor in s390/util/auxtrace.c when REFCNT_CHECKING is enabled, as well as the need for additional perf_sample__exit() cleanups in C core error paths (e.g. builtin-inject.c) and NULL checks for evsel__get() after evlist__id2evsel(), are great catches. These C-core adjustments will be handled in follow-up patches as we harden the reference counting baseline in Phase 2.
- **Concurrency & Lifecycles:** The potential for a double-free race in evlist__put() under concurrent destruction is a known limitation. Implementing a thread-safe, symmetrical global GC lifecycle is a core goal of Phase 2.
- **API Boundary Hardening:** The need for PyObject_TypeCheck validation in pyrf_evsel__open() (similar to what we added for parse_events in v21) will be addressed as we continue to lock down the C-API boundary in Phase 2.
- **Stubs & Scripts:** The refinement of the perf.pyi typing (e.g., handling the MetricGroup list of strings) and the actual migration of syscall-counts.py to use the new syscall_name/id methods are explicitly the focus of Phase 2 ("Script Porting & Tool Migration").
- **Cross-endian analysis:** As noted in the cover letter, proper cross-endian analysis for packed 32-bit fields is deferred to Phase 2.
I'm looking forward to landing this Phase 1 baseline so we can begin sending the Phase 2 series to address these remaining architectural updates and script migrations.
Thanks,
Ian
Quoted 305 lines · Ian Rogers
The perf script command has long supported running Python and Perl scripts by embedding libpython and libperl. This approach has several drawbacks:
- overhead by creating Python dictionaries for every event (whether used or not),
- complex build dependencies on specific Python/Perl versions,
- complications with threading due to perf being the interpreter,
- no clear way to run standalone scripts likeilist.py.This series takes a different approach with some initial implementation posted as an RFC last October:
https://lore.kernel.org/linux-perf-users/[email protected]/It builds the python extension as part of the normal build. The extension is able to read perf.data files. The event callbacks are converted to have a python evsel/evlist/sample passed to them.
To make the review process more manageable, the original 58-patch series has been split. This v18 series represents "Phase 1: API & Infrastructure" (20 patches). The first 4 patches of Phase 1 (cleanups and arch-specific header sorting) have already been merged upstream.
This remaining set contains:
1. Missed explicit dependency cleanups and header sorting for util/ and python.
2. Crucial core safety infrastructure (reference counting for evlist/evsel) to support safe lifecycle management in garbage-collected Python.
3. The core Python API extensions (session wrappers,perf_datawrappers, sample accessors, stubs, and LiveSession helper).Phase 2 ("Script Porting & Tool Migration") will migrate the remaining 35+ existing Python/Perl scripts to the new API (which yields up to 35x speedups as demonstrated previously) and the final removal of embedded interpreters.
Addressing v20 Review Feedback:
The v21 patches merely update a commit message. Sashiko has correctly identified several structural limitations and edge cases. Almost all of these points correspond to known limitations that we have explicitly mapped out and planned for "Phase 2" of this Python binding refactor. Our primary goal with this v21 series (Phase 1) is to establish the baseline abstractions, fix immediate memory corruption bugs, and provide the foundational wrappers.
- Patch 4 (perf data: Add open flag): The reported issue with
perf_data__switchfailing to close thread-local files during directory mode rotation is a pre-existing bug in the C core. This patch strictly fixes the booleandata->openstate tracking for Python integration.- Patch 7 (perf evlist: Add reference count checking): evlist is fundamentally not thread-safe for concurrent destruction in the C core. The asymmetric leak (leaving cycles intact if evlist is dropped before evsel) is a known limitation. Implementing a thread-safe, symmetrical global GC lifecycle for evlist/evsel is planned for Phase 2.
- Patch 10 (perf python: Add python session abstraction):
1) perf.thread cannot cause a NULL dereference because it does not implement tp_new/tp_init, so it cannot be instantiated directly from Python. 2) Proper cross-endian analysis of packed 32-bit fields and the implementation of the remaining tool callbacks are explicitly deferred to Phase 2.- Patch 11 (perf python: Refactor and add accessors to sample event):
1) Session UAF due to tracking the topological lifetime of the session relative to events requires a major lifecycle overhaul, slated for Phase 2. 2) The commit message previously inaccurately claimed we only allocate the strictly necessary copy size. This has been corrected in v21. We usePyObject_Newwith the statictp_basicsize; optimizing the dynamic allocation size is a future cleanup.- Patch 16 (perf python: Add syscall name/id): The legacy scripts remain untouched in this series. Migrating the existing Python/Perl scripts to use the new C-API and dropping libaudit entirely is the core objective of Phase 2.
- Patch 20 (perf python: Add perf.pyi stubs file): While the stub is currently loosely typed as Any for threads, the segmentation fault is no longer possible since Patch 19 added runtime type validation (
PyObject_TypeCheck) to the C implementation. We will tighten the stub typing in a follow-up patch.Addressing v19 Review Feedback:
- Patch 19: AddedPyObject_TypeCheckruntime validations toparse_eventsandparse_metricsto prevent memory corruption when invalid objects are passed from Python, resolving the blind C cast vulnerability.
- Patch 20: Updated perf.pyi stubs to properly type the threads parameter as Optional['thread_map'] instead of Optional[Any] to catch invalid types during static analysis.Note: Other architectural limitations raised in the v19 review (e.g. TOCTOU cycle races, asymmetric cycle leaks, cross-endian
needs_swapbypass, and session object dangling pointers) are explicitly acknowledged as limitations of this transitional patch set. As noted previously, implementing thread-safe, symmetrical GC for the Python bindings and hardening the C-API boundary are the primary focus of the upcoming Phase 2 series.Addressing v18 Review Feedback:
- Patch 10 (perf.threadinitialization): Added missingCHECK_INITIALIZED()inpyrf_thread__commto prevent NULL dereference when instantiated directly.
- Patch 14 (STAT events): PassedNULLtopyrf_event__newfor STAT events to prevent unconditionalevsel__parse_sampleand potential out-of-bounds reads.
- Patch 20 (LiveSessiontimeout): Broadened exception handling to explicitly ignore "Unexpected header type" for valid but unsupported events.Note: Other feedback items raised in v18 (TOCTOU cycle races, asymmetric cycle collection, cross-endian data handling, guest machine symbol resolution, and pre-existing memory leaks/uninitialized variables) are acknowledged as limitations of the current implementation and will be addressed in Phase 2 or separate cleanup patches.
Addressing v17 Review Feedback:
- Patch 8: Added missingperf_sample__exit(&sample)tointel_pt_synth_ptwrite_sample()to fix an evsel reference leak.
- Patch 10: Fixed a bug inpyrf_session_tool__sample()that caused double byte-swapping on foreign-endian files by temporarily disablingneeds_swapduring re-parsing instead of assigning*sample.
- Patch 11: (Missed fixing address resolution for guest samples - will fix in next spin or Phase 2).
- Patch 19: Preventedpython-cleanfrom deleting tracked source filepython/perf.pyiwhen building in tree. Also explicitly exportedCOUNT_HW_REF_CPU_CYCLES,thread,callchain, andcallchain_nodetypes/constants to the module viaPyInit_perf.
- Patch 20: Narrowedexcept TypeErrorinLiveSession.run()to explicitly check for "Unknown CPU" so legitimate event parsing failures aren't swallowed.Addressing v16 Review Feedback:
- Patch 10: Removed unconditionalperf_session__create_kernel_mapsto prevent corrupting cross-platform offline analysis.
- Patch 11: Corrected inaccurate commit message regarding memory allocation sizing.
- Patch 19: Fixed numerous type inconsistencies, missing properties, and incorrect return types in theperf.pyistubs file.
- Patch 20: Cleaned up unusedimport errnoinperf_live.py.Note: Several issues spotted in v16/v17 review (e.g.
pyrf_evsel__initformat string type mismatch, evlist lockless double free, asymmetric memory leaks, missingPy_Nonetype checks, and lack of NUL-termination for COMM/MMAP) are pre-existing limitations in the codebase or side-effects of the transitional cycle-breaking design. As discussed previously, these structurally complex or pre-existing bugs are deliberately deferred to the Phase 2 series.Addressing v15 Review Feedback:
- Patch 2 (buffer overflow & type checks): The buffer overflow inpyrf_event__new()has been resolved by verifyingevent->header.sizeagainst the event struct size.
- Patch 2 (PyObject_HEAD_INIT): The initialization macros have been corrected to use the proper Python 3 compatibility approach.
- Patch 5 (Memory leak & RC validation): Applied extensive structural fixes usingrefcount_tsemantics. Added validation wrapper structures to statically verify memory access safety in the lockless cycles.
- Patch 8 (evlist.openmemory leak): Restructured lifecycle management for mmap buffers usingdo_munmap()hooks inevlist__put().
- Patch 16 (perf.pyistubs): Corrected return types (Optionaland proper object properties) and missing documentation strings in type stubs.
- Patch 20 (perf_live.pytimeout): Adjusted poll timeout from 10000ms back to 100ms, replacing the tight exception loop.Ian Rogers (21):
perf util: Sort includes and add missed explicit dependencies perf python: Add missed explicit dependencies
perf evsel/evlist: Avoid unnecessary #includes
perf data: Add open flag
perf evlist: Add reference count
perf evsel: Add reference count
perf evlist: Add reference count checking
perf python: Use evsel in sample inpyrf_event
perf python: Add wrapper forperf_datafile abstraction perf python: Add python session abstraction wrapping perf's session perf python: Refactor and add accessors to sample event perf python: Add mmap2 event
perf python: Add callchain support
perf python: Extend API for stat events inpython.c
perf python: Expose brstack in sample event
perf python: Add syscall name/id to convert syscall number and name perf python: Add config file access
perf python: HandlePy_Nonefor thread and cpu maps
perf python: Add type checking for parse_events/parse_metrics perf python: Add perf.pyi stubs file
perf python: Add LiveSession helpertools/perf/Makefile.perf | 7 +- tools/perf/arch/arm/util/cs-etm.c | 10 +- tools/perf/arch/arm64/util/arm-spe.c | 8 +- tools/perf/arch/arm64/util/hisi-ptt.c | 2 +- tools/perf/arch/x86/tests/hybrid.c | 22 +- tools/perf/arch/x86/tests/topdown.c | 4 +- tools/perf/arch/x86/util/auxtrace.c | 2 +- tools/perf/arch/x86/util/intel-bts.c | 6 +- tools/perf/arch/x86/util/intel-pt.c | 9 +- tools/perf/arch/x86/util/iostat.c | 14 +- tools/perf/bench/evlist-open-close.c | 29 +- tools/perf/builtin-annotate.c | 7 +- tools/perf/builtin-ftrace.c | 14 +- tools/perf/builtin-inject.c | 9 +- tools/perf/builtin-kvm.c | 14 +- tools/perf/builtin-kwork.c | 8 +- tools/perf/builtin-lock.c | 4 +- tools/perf/builtin-record.c | 95 +- tools/perf/builtin-report.c | 6 +- tools/perf/builtin-sched.c | 30 +- tools/perf/builtin-script.c | 15 +- tools/perf/builtin-stat.c | 83 +- tools/perf/builtin-top.c | 104 +- tools/perf/builtin-trace.c | 65 +- tools/perf/python/perf.pyi | 672 +++++ tools/perf/python/perf_live.py | 59 + tools/perf/tests/backward-ring-buffer.c | 26 +- tools/perf/tests/code-reading.c | 14 +- tools/perf/tests/event-times.c | 6 +- tools/perf/tests/event_update.c | 4 +- tools/perf/tests/evsel-roundtrip-name.c | 8 +- tools/perf/tests/evsel-tp-sched.c | 4 +- tools/perf/tests/expand-cgroup.c | 12 +- tools/perf/tests/hists_cumulate.c | 2 +- tools/perf/tests/hists_filter.c | 2 +- tools/perf/tests/hists_link.c | 2 +- tools/perf/tests/hists_output.c | 2 +- tools/perf/tests/hwmon_pmu.c | 7 +- tools/perf/tests/keep-tracking.c | 10 +- tools/perf/tests/mmap-basic.c | 24 +- tools/perf/tests/openat-syscall-all-cpus.c | 6 +- tools/perf/tests/openat-syscall-tp-fields.c | 26 +- tools/perf/tests/openat-syscall.c | 6 +- tools/perf/tests/parse-events.c | 139 +- tools/perf/tests/parse-metric.c | 8 +- tools/perf/tests/parse-no-sample-id-all.c | 2 +- tools/perf/tests/perf-record.c | 38 +- tools/perf/tests/perf-time-to-tsc.c | 12 +- tools/perf/tests/pfm.c | 12 +- tools/perf/tests/pmu-events.c | 11 +- tools/perf/tests/pmu.c | 4 +- tools/perf/tests/sample-parsing.c | 45 +- tools/perf/tests/shell/lib/setup_python.sh | 13 + tools/perf/tests/sw-clock.c | 20 +- tools/perf/tests/switch-tracking.c | 11 +- tools/perf/tests/task-exit.c | 20 +- tools/perf/tests/time-utils-test.c | 14 +- tools/perf/tests/tool_pmu.c | 7 +- tools/perf/tests/topology.c | 4 +- tools/perf/tests/uncore-event-sorting.c | 6 +- tools/perf/ui/browsers/annotate.c | 2 +- tools/perf/ui/browsers/hists.c | 22 +- tools/perf/util/Build | 1 - tools/perf/util/amd-sample-raw.c | 2 +- tools/perf/util/annotate-data.c | 2 +- tools/perf/util/annotate.c | 10 +- tools/perf/util/auxtrace.c | 14 +- tools/perf/util/block-info.c | 4 +- tools/perf/util/bpf_counter.c | 2 +- tools/perf/util/bpf_counter_cgroup.c | 14 +- tools/perf/util/bpf_ftrace.c | 9 +- tools/perf/util/bpf_lock_contention.c | 12 +- tools/perf/util/bpf_off_cpu.c | 44 +- tools/perf/util/bpf_trace_augment.c | 8 +- tools/perf/util/cgroup.c | 26 +- tools/perf/util/cs-etm.c | 5 +- tools/perf/util/data-convert-bt.c | 2 +- tools/perf/util/data.c | 27 +- tools/perf/util/data.h | 4 +- tools/perf/util/evlist.c | 496 ++-- tools/perf/util/evlist.h | 273 +- tools/perf/util/evsel.c | 39 +- tools/perf/util/evsel.h | 40 +- tools/perf/util/expr.c | 2 +- tools/perf/util/header.c | 69 +- tools/perf/util/header.h | 2 +- tools/perf/util/intel-pt.c | 8 +- tools/perf/util/intel-tpebs.c | 7 +- tools/perf/util/iostat.c | 2 +- tools/perf/util/iostat.h | 2 +- tools/perf/util/map.h | 9 +- tools/perf/util/metricgroup.c | 12 +- tools/perf/util/parse-events.c | 10 +- tools/perf/util/parse-events.y | 2 +- tools/perf/util/perf_api_probe.c | 20 +- tools/perf/util/pfm.c | 4 +- tools/perf/util/print-events.c | 2 +- tools/perf/util/python.c | 2846 ++++++++++++++++--- tools/perf/util/record.c | 11 +- tools/perf/util/s390-sample-raw.c | 20 +- tools/perf/util/sample-raw.c | 4 +- tools/perf/util/sample.c | 17 +- tools/perf/util/session.c | 69 +- tools/perf/util/session.h | 2 + tools/perf/util/setup.py | 5 + tools/perf/util/sideband_evlist.c | 40 +- tools/perf/util/sort.c | 2 +- tools/perf/util/stat-display.c | 6 +- tools/perf/util/stat-shadow.c | 24 +- tools/perf/util/stat.c | 20 +- tools/perf/util/stream.c | 4 +- tools/perf/util/synthetic-events.c | 11 +- tools/perf/util/time-utils.c | 12 +- tools/perf/util/top.c | 4 +- 114 files changed, 4602 insertions(+), 1529 deletions(-) create mode 100644 tools/perf/python/perf.pyi create mode 100755 tools/perf/python/perf_live.py--
2.54.0.1136.gdb2ca164c4-goog
Ian Rogers
reviewerPATCH v4 28/58 perf syscall-counts-by-pid: Port syscall-counts-by-pid to use python module
In reply to Ian RogersRewrite tools/perf/scripts/python/syscall-counts-by-pid.py to use the python module and various style changes. By avoiding the overheads in the perf script execution the performance improves by more than 3.8x as shown in the following (with PYTHON_PATH and PERF_EXEC_PATH set as necessary):
``` $ perf record -e raw_syscalls:sys_enter -a sleep 1 ... $ time perf script tools/perf/scripts/python/syscall-counts-by-pid.py perf Install the python-audit package to get syscall names. For example: # apt-get install python3-audit (Ubuntu) # yum install python3-audit (Fedora) etc. Press control+C to stop and show the summary Warning: 1 out of order events recorded. syscall events for perf: comm [pid]/syscalls count --------------------------------------- ---------- perf [3886080] 1 538989 16 32 203 17 3 2 257 1 204 1 15 1 0 1 perf [3886082] 7 1 real 0m3.852s user 0m3.512s sys 0m0.336s $ time python3 tools/perf/python/syscall-counts-by-pid.py perf Warning: 1 out of order events recorded. syscall events for perf: comm [pid]/syscalls count --------------------------------------- ----------- perf [3886080] write 538989 ioctl 32 sched_setaffinity 17 close 2 openat 1 sched_getaffinity 1 rt_sigreturn 1 read 1 perf [3886082] poll 1 real 0m1.011s user 0m0.963s sys 0m0.048s ```
---
v2:
1. Removed Unused Variable: Removed id_keys which was assigned but never read.
2. Fallback for Unknown Syscalls: If perf.syscall_name() returns None for an unmapped ID, it now falls back to the numeric ID string to prevent TypeError crashes during string formatting.
3. Fallback for Syscall Number Attribute: It now checks for __syscall_nr first, and if missing, falls back to checking for nr .
4. Robust Process Resolution: Added a try-except block around session.process(sample.pid).comm() to handle untracked PIDs gracefully instead of crashing on a TypeError .
5. Restored PID Filtering: The script now attempts to parse the positional argument as an integer to filter by Process ID. If that fails, it treats it as a command name (COMM) string to filter by, restoring behavior from the original legacy script.
6. Support for Custom Input Files: Added a -i / --input command-line argument to support arbitrarily named trace files, removing the hardcoded "perf.data" restriction.
--- tools/perf/python/syscall-counts-by-pid.py | 88 ++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 tools/perf/python/syscall-counts-by-pid.py
tools/perf/python/syscall-counts-by-pid.py new file +88 -0
@@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Displays system-wide system call totals, broken down by syscall. +If a [comm] arg is specified, only syscalls called by [comm] are displayed. +""" + +import argparse +from collections import defaultdict +import perf + +syscalls: dict[tuple[str, int, int], int] = defaultdict(int) +for_comm = None +for_pid = None +session = None + + +def print_syscall_totals(): + """Print aggregated statistics.""" + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + elif for_pid is not None: + print(f"\nsyscall events for PID {for_pid}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'comm [pid]/syscalls':<40} {'count':>10}") + print("---------------------------------------- -----------") + + sorted_keys = sorted(syscalls.keys(), key=lambda k: (k[0], k[1], k[2])) + current_comm_pid = None + for comm, pid, sc_id in sorted_keys: + if current_comm_pid != (comm, pid): + print(f"\n{comm} [{pid}]") + current_comm_pid = (comm, pid) + name = perf.syscall_name(sc_id) or str(sc_id) + print(f" {name:<38} {syscalls[(comm, pid, sc_id)]:>10}") + + +def process_event(sample): + """Process a single sample event.""" + event_name = str(sample.evsel) + if event_name == "evsel(raw_syscalls:sys_enter)": + sc_id = getattr(sample, "id", -1) + elif event_name.startswith("evsel(syscalls:sys_enter_"): + sc_id = getattr(sample, "__syscall_nr", None) + if sc_id is None: + sc_id = getattr(sample, "nr", -1) + else: + return + + if sc_id == -1: + return + + pid = sample.sample_pid + + if for_pid and pid != for_pid: + return + + comm = "unknown" + try: + if session: + proc = session.process(pid) + if proc: + comm = proc.comm() + except (TypeError, AttributeError): + pass + + if for_comm and comm != for_comm: + return + syscalls[(comm, pid, sc_id)] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("filter", nargs="?", help="COMM or PID to filter by") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + if args.filter: + try: + for_pid = int(args.filter) + except ValueError: + for_comm = args.filter + + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + print_syscall_totals()
Signature · 1 line
2.54.0.rc2.533.g4f5dca5207-goog
Ian Rogers
reviewerPATCH v4 29/58 perf futex-contention: Port futex-contention to use python module
Rewrite tools/perf/scripts/python/futex-contention.py to use the python module and various style changes. By avoiding the overheads in the perf script execution the performance improves by more than 3.2x as shown in the following (with PYTHON_PATH and PERF_EXEC_PATH set as necessary):
``` $ perf record -e syscalls:sys_*_futex -a sleep 1 ... $ time perf script tools/perf/scripts/python/futex-contention.py Install the python-audit package to get syscall names. For example: # apt-get install python3-audit (Ubuntu) # yum install python3-audit (Fedora) etc. Press control+C to stop and show the summary aaa/4[2435653] lock 7f76b380c878 contended 1 times, 1099 avg ns [max: 1099 ns, min 1099 ns] ... real 0m1.007s user 0m0.935s sys 0m0.072s $ time python3 tools/perf/python/futex-contention.py ... real 0m0.314s user 0m0.259s sys 0m0.056s ```
---
v2:
1. Fixed Module Import Failure: Corrected the type annotations from [int, int] to Tuple[int, int] . The previous code would raise a TypeError at module import time because lists cannot be used as types in dictionary annotations.
2. Prevented Out-Of-Memory Crashes: Replaced the approach of storing every single duration in a list with a LockStats class that maintains running aggregates (count, total time, min, max). This ensures O(1) memory usage per lock/thread pair rather than unbounded memory growth.
3. Support for Custom Input Files: Added a -i / --input command-line argument to support processing arbitrarily named trace files, removing the hardcoded "perf.data" restriction.
4. Robust Process Lookup: Added a check to ensure session is initialized before calling session. process() , preventing potential NoneType attribute errors if events are processed during initialization.
--- tools/perf/python/futex-contention.py | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 tools/perf/python/futex-contention.py
tools/perf/python/futex-contention.py new file +87 -0
@@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Measures futex contention.""" + +import argparse +from collections import defaultdict +from typing import Dict, Tuple +import perf + +class LockStats: + """Aggregate lock contention information.""" + def __init__(self) -> None: + self.count = 0 + self.total_time = 0 + self.min_time = 0 + self.max_time = 0 + + def add(self, duration: int) -> None: + """Add a new duration measurement.""" + self.count += 1 + self.total_time += duration + if self.count == 1: + self.min_time = duration + self.max_time = duration + else: + self.min_time = min(self.min_time, duration) + self.max_time = max(self.max_time, duration) + + def avg(self) -> float: + """Return average duration.""" + return self.total_time / self.count if self.count > 0 else 0.0 + +process_names: Dict[int, str] = {} +start_times: Dict[int, Tuple[int, int]] = {} +session = None +durations: Dict[Tuple[int, int], LockStats] = defaultdict(LockStats) + +FUTEX_WAIT = 0 +FUTEX_WAKE = 1 +FUTEX_PRIVATE_FLAG = 128 +FUTEX_CLOCK_REALTIME = 256 +FUTEX_CMD_MASK = ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) + + +def process_event(sample: perf.sample_event) -> None: + """Process a single sample event.""" + def handle_start(tid: int, uaddr: int, op: int, start_time: int) -> None: + if (op & FUTEX_CMD_MASK) != FUTEX_WAIT: + return + if tid not in process_names: + try: + if session: + process = session.process(tid) + if process: + process_names[tid] = process.comm() + except (TypeError, AttributeError): + return + start_times[tid] = (uaddr, start_time) + + def handle_end(tid: int, end_time: int) -> None: + if tid not in start_times: + return + (uaddr, start_time) = start_times[tid] + del start_times[tid] + durations[(tid, uaddr)].add(end_time - start_time) + + event_name = str(sample.evsel) + if event_name == "evsel(syscalls:sys_enter_futex)": + uaddr = getattr(sample, "uaddr", 0) + op = getattr(sample, "op", 0) + handle_start(sample.sample_tid, uaddr, op, sample.sample_time) + elif event_name == "evsel(syscalls:sys_exit_futex)": + handle_end(sample.sample_tid, sample.sample_time) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Measure futex contention") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + + for ((t, u), stats) in sorted(durations.items()): + avg_ns = stats.avg() + print(f"{process_names.get(t, 'unknown')}[{t}] lock {u:x} contended {stats.count} times, " + f"{avg_ns:.0f} avg ns [max: {stats.max_time} ns, min {stats.min_time} ns]")
Signature · 1 line
2.54.0.rc2.533.g4f5dca5207-goog
Ian Rogers
reviewerPATCH v4 30/58 perf flamegraph: Port flamegraph to use python module
In reply to Ian RogersAdd a port of the flamegraph script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
---
v2:
1. Performance Optimization: Changed Node.children from a list to a dictionary, reducing the lookup time in find_or_create_node from O(N) to O(1) and avoiding performance bottlenecks on wide call graphs.
2. Callchain Fallback: Added a fallback to use the sample's top-level symbol or instruction pointer if no callchain is present, ensuring the script still generates meaningful output rather than just process names.
3. Template Downloading Fix: Corrected the logic handling the --allow-download flag and custom HTTP URLs. It no longer warns about missing local files when a URL is provided, and won't silently overwrite custom URLs with the default one.
4. Output Stream Separation: Moved informational warnings to sys.stderr to prevent them from corrupting the resulting HTML/JSON file when the user streams the output to stdout (e.g., using -o - ).
5. XSS Protection: Added basic HTML entity escaping for < , > , and & within the embedded JSON data blocks. This mitigates the risk of cross-site scripting if trace data contains maliciously formed process or symbol names.
--- tools/perf/python/flamegraph.py | 250 ++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100755 tools/perf/python/flamegraph.py
tools/perf/python/flamegraph.py new file +250 -0
@@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +flamegraph.py - create flame graphs from perf samples using perf python module +""" + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import urllib.request +from typing import Dict, Optional, Union +import perf + +MINIMAL_HTML = """<head> + <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/d3-flamegraph.css"> +</head> +<body> + <div id="chart"></div> + <script type="text/javascript" src="https://d3js.org/d3.v7.js"></script> + <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/[email protected]/dist/d3-flamegraph.min.js"></script> + <script type="text/javascript"> + const stacks = [/** @flamegraph_json **/]; + // Note, options is unused. + const options = [/** @options_json **/]; + + var chart = flamegraph(); + d3.select("#chart") + .datum(stacks[0]) + .call(chart); + </script> +</body> +""" + +class Node: + """A node in the flame graph tree.""" + def __init__(self, name: str, libtype: str): + self.name = name + self.libtype = libtype + self.value: int = 0 + self.children: dict[str, Node] = {} + + def to_json(self) -> Dict[str, Union[str, int, list[Dict]]]: + """Convert the node to a JSON-serializable dictionary.""" + return { + "n": self.name, + "l": self.libtype, + "v": self.value, + "c": [x.to_json() for x in self.children.values()] + } + + +class FlameGraphCLI: + """Command-line interface for generating flame graphs.""" + def __init__(self, args): + self.args = args + self.stack = Node("all", "root") + self.session = None + + @staticmethod + def get_libtype_from_dso(dso: Optional[str]) -> str: + """Determine the library type from the DSO name.""" + if dso and (dso == "[kernel.kallsyms]" or dso.endswith("/vmlinux") or dso == "[kernel]"): + return "kernel" + return "" + + @staticmethod + def find_or_create_node(node: Node, name: str, libtype: str) -> Node: + """Find a child node with the given name or create a new one.""" + if name in node.children: + return node.children[name] + child = Node(name, libtype) + node.children[name] = child + return child + + def process_event(self, sample) -> None: + """Process a single perf sample event.""" + if self.args.event_name and str(sample.evsel) != self.args.event_name: + return + + pid = sample.sample_pid + dso_type = "" + try: + thread = self.session.process(sample.sample_tid) + comm = thread.comm() + except Exception: + comm = "[unknown]" + + if pid == 0: + comm = "swapper" + dso_type = "kernel" + else: + comm = f"{comm} ({pid})" + + node = self.find_or_create_node(self.stack, comm, dso_type) + + callchain = sample.callchain + if callchain: + # We want to traverse from root to leaf. + # perf callchain iterator gives leaf to root. + # We collect them and reverse. + frames = list(callchain) + for entry in reversed(frames): + name = entry.symbol or "[unknown]" + libtype = self.get_libtype_from_dso(entry.dso) + node = self.find_or_create_node(node, name, libtype) + else: + # Fallback if no callchain + name = getattr(sample, "symbol", "[unknown]") + libtype = self.get_libtype_from_dso(getattr(sample, "dso", "[unknown]")) + node = self.find_or_create_node(node, name, libtype) + + node.value += 1 + + def get_report_header(self) -> str: + """Get the header from the perf report.""" + try: + input_file = self.args.input or "perf.data" + output = subprocess.check_output(["perf", "report", "--header-only", "-i", input_file]) + result = output.decode("utf-8") + if self.args.event_name: + result += "\nFocused event: " + self.args.event_name + return result + except Exception: + return "" + + def run(self) -> None: + """Run the flame graph generation.""" + input_file = self.args.input or "perf.data" + if not os.path.exists(input_file): + print(f"Error: {input_file} not found. (try 'perf record' first)", file=sys.stderr) + sys.exit(1) + + try: + self.session = perf.session(perf.data(input_file), + sample=self.process_event) + except Exception as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + self.session.process_events() + + stacks_json = json.dumps(self.stack, default=lambda x: x.to_json()) + # Escape HTML special characters to prevent XSS + stacks_json = stacks_json.replace("<", "\\u003c") \ + .replace(">", "\\u003e").replace("&", "\\u0026") + + if self.args.format == "html": + report_header = self.get_report_header() + options = { + "colorscheme": self.args.colorscheme, + "context": report_header + } + options_json = json.dumps(options) + options_json = options_json.replace("<", "\\u003c") \ + .replace(">", "\\u003e").replace("&", "\\u0026") + + template = self.args.template + template_md5sum = None + output_str = None + + if not os.path.isfile(template): + if template.startswith("http://") or template.startswith("https://"): + if not self.args.allow_download: + print("Warning: Downloading templates is disabled. " + "Use --allow-download.", file=sys.stderr) + template = None + else: + print(f"Warning: Template file '{template}' not found.", file=sys.stderr) + if self.args.allow_download: + print("Using default CDN template.", file=sys.stderr) + template = ( + "https://cdn.jsdelivr.net/npm/[email protected]/dist/templates/" + "d3-flamegraph-base.html" + ) + template_md5sum = "143e0d06ba69b8370b9848dcd6ae3f36" + else: + template = None + + use_minimal = False + try: + if not template: + use_minimal = True + elif template.startswith("http"): + with urllib.request.urlopen(template) as url_template: + output_str = "".join([l.decode("utf-8") for l in url_template.readlines()]) + else: + with open(template, "r", encoding="utf-8") as f: + output_str = f.read() + except Exception as err: + print(f"Error reading template {template}: {err}\n", file=sys.stderr) + use_minimal = True + + if use_minimal: + print("Using internal minimal HTML that refers to d3's web site. JavaScript " + + "loaded this way from a local file may be blocked unless your " + + "browser has relaxed permissions. Run with '--allow-download' to fetch" + + "the full D3 HTML template.", file=sys.stderr) + output_str = MINIMAL_HTML + + elif template_md5sum: + assert output_str is not None + download_md5sum = hashlib.md5(output_str.encode("utf-8")).hexdigest() + if download_md5sum != template_md5sum: + s = None + while s not in ["y", "n"]: + s = input(f"""Unexpected template md5sum. +{download_md5sum} != {template_md5sum}, for: +{output_str} +continue?[yn] """).lower() + if s == "n": + sys.exit(1) + + assert output_str is not None + output_str = output_str.replace("/** @options_json **/", options_json) + output_str = output_str.replace("/** @flamegraph_json **/", stacks_json) + output_fn = self.args.output or "flamegraph.html" + else: + output_str = stacks_json + output_fn = self.args.output or "stacks.json" + + if output_fn == "-": + sys.stdout.write(output_str) + else: + print(f"dumping data to {output_fn}") + with open(output_fn, "w", encoding="utf-8") as out: + out.write(output_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Create flame graphs using perf python module.") + parser.add_argument("-f", "--format", default="html", choices=["json", "html"], + help="output file format") + parser.add_argument("-o", "--output", help="output file name") + parser.add_argument("--template", + default="/usr/share/d3-flame-graph/d3-flamegraph-base.html", + help="path to flame graph HTML template") + parser.add_argument("--colorscheme", default="blue-green", + help="flame graph color scheme", choices=["blue-green", "orange"]) + parser.add_argument("-i", "--input", help="input perf.data file") + parser.add_argument("--allow-download", default=False, action="store_true", + help="allow unprompted downloading of HTML template") + parser.add_argument("-e", "--event", default="", dest="event_name", type=str, + help="specify the event to generate flamegraph for") + + cli_args = parser.parse_args() + cli = FlameGraphCLI(cli_args) + cli.run()
Signature · 1 line
2.54.0.rc2.533.g4f5dca5207-goog
Ian Rogers
reviewerPATCH 27/58 perf syscall-counts: Port syscall-counts to use python module
In reply to Ian RogersRewrite tools/perf/scripts/python/syscall-counts.py to use the python module and various style changes. By avoiding the overheads in the perf script execution the performance improves by more than 4x as shown in the following (with PYTHON_PATH and PERF_EXEC_PATH set as necessary):
``` $ perf record -e raw_syscalls:sys_enter -a sleep 1 ... $ time perf script tools/perf/scripts/python/syscall-counts.py perf Install the python-audit package to get syscall names. For example: # apt-get install python3-audit (Ubuntu) # yum install python3-audit (Fedora) etc. Press control+C to stop and show the summary Warning: 1 out of order events recorded. syscall events for perf: event count -------------------------------------- ------------ 1 538989 16 32 203 17 3 2 257 1 204 1 15 1 7 1 0 1 real 0m3.887s user 0m3.578s sys 0m0.308s $ time python3 tools/perf/python/syscall-counts.py perf Warning: 1 out of order events recorded. syscall events for perf: event count -------------------------------------- ------------ write 538989 ioctl 32 sched_setaffinity 17 close 2 openat 1 sched_getaffinity 1 rt_sigreturn 1 poll 1 read 1 real 0m0.953s user 0m0.905s sys 0m0.048s ```
--- tools/perf/python/syscall-counts.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 tools/perf/python/syscall-counts.py
tools/perf/python/syscall-counts.py new file +49 -0
@@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Displays system-wide system call totals, broken down by syscall. +# If a [comm] arg is specified, only syscalls called by [comm] are displayed. + +import argparse +from collections import defaultdict +import perf + +syscalls = defaultdict(int) +for_comm = None +session = None + + +def print_syscall_totals(): + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'event':<40} {'count':>10}") + print("---------------------------------------- -----------") + + for id, val in sorted(syscalls.items(), + key=lambda kv: (kv[1], kv[0]), reverse=True): + print(f"{perf.syscall_name(id):<40} {val:>10}") + + +def process_event(sample): + event_name = str(sample.evsel) + if event_name == "evsel(raw_syscalls:sys_enter)": + id = sample.id + elif event_name.startswith("evsel(syscalls:sys_enter_"): + id = sample.__syscall_nr + else: + return + if for_comm and session.process(sample.sample_pid).comm() != for_comm: + return + syscalls[id] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("comm", nargs="?", help="Only report syscalls for comm") + args = ap.parse_args() + for_comm = args.comm + session = perf.session(perf.data("perf.data"), sample=process_event) + session.process_events() + print_syscall_totals()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 27/58] perf syscall-counts: Port syscall-counts to use python module
Sashiko AI review found 4 potential issue(s): - [High] The script crashes with a TypeError when a thread lookup fails during 'for_comm' filtering. - [High] The script crashes with a TypeError if…
Sashiko AI review found 4 potential issue(s):
- [High] The script crashes with a TypeError when a thread lookup fails during 'for_comm' filtering.
- [High] The script crashes with a TypeError if 'perf.syscall_name(id)' encounters an unmapped syscall ID.
- [High] The script crashes with an AttributeError on older kernels where the tracepoint field is 'nr' instead of '__syscall_nr'.
- [Medium] The standalone script hardcodes the input trace file to 'perf.data', breaking the ability to analyze arbitrarily named trace files. --
commit 6f15d3f04b68eebb0fdc02fb0dd260ac9c6fdf6a
Author: Ian Rogers <[email protected]>
perf syscall-counts: Port syscall-counts to use python module
This commit rewrites the tools/perf/scripts/python/syscall-counts.py script to use the python module directly. This avoids the overhead of executing via perf script, improving execution performance.
Quoted 5 lines
tools/perf/python/syscall-counts.py new file +0
-0
[ ... ]
tools/perf/python/syscall-counts.py line 26
Quoted 12 lines
tools/perf/python/syscall-counts.py new file +12
-0+def print_syscall_totals(): + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'event':<40} {'count':>10}") + print("---------------------------------------- -----------") + + for id, val in sorted(syscalls.items(), + key=lambda kv: (kv[1], kv[0]), reverse=True): + print(f"{perf.syscall_name(id):<40} {val:>10}")
What happens if perf.syscall_name(id) returns None for an unmapped syscall ID?
Applying a width specifier (<40) to a NoneType object throws a TypeError, which will abort the output. Should there be a fallback mechanism, such as using the numeric ID when the name is unknown?
tools/perf/python/syscall-counts.py line 34
Quoted 6 lines
tools/perf/python/syscall-counts.py new file +6
-0+def process_event(sample): + event_name = str(sample.evsel) + if event_name == "evsel(raw_syscalls:sys_enter)": + id = sample.id + elif event_name.startswith("evsel(syscalls:sys_enter_"): + id = sample.__syscall_nr
On some older kernels and architectures, the tracepoint format field for syscalls is named nr instead of __syscall_nr.
Will accessing sample.__syscall_nr raise an AttributeError in those environments? Is it possible to check for nr as a fallback?
tools/perf/python/syscall-counts.py line 37
Quoted 3 lines
tools/perf/python/syscall-counts.py new file +3
-0+ else: + return + if for_comm and session.process(sample.sample_pid).comm() != for_comm:
If a thread lookup fails (for example, for short-lived processes or samples recorded before procfs synthesis completes), session.process() returns NULL and sets a Python TypeError.
Can this uncaught exception bubble up and cause the script to crash?
tools/perf/python/syscall-counts.py line 47
Quoted 10 lines
tools/perf/python/syscall-counts.py new file +10
-0+ return + syscalls[id] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("comm", nargs="?", help="Only report syscalls for comm") + args = ap.parse_args() + for_comm = args.comm + session = perf.session(perf.data("perf.data"), sample=process_event)
The previous implementation running under perf script automatically supported standard options like -i my_trace.data to override the input file.
Since the file path is now hardcoded to perf.data, does this introduce a regression in usability by preventing the analysis of arbitrarily named trace files?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=27
Ian Rogers
reviewerPATCH 28/58 perf syscall-counts-by-pid: Port syscall-counts-by-pid to use python module
In reply to Ian RogersRewrite tools/perf/scripts/python/syscall-counts-by-pid.py to use the python module and various style changes. By avoiding the overheads in the perf script execution the performance improves by more than 3.8x as shown in the following (with PYTHON_PATH and PERF_EXEC_PATH set as necessary):
``` $ perf record -e raw_syscalls:sys_enter -a sleep 1 ... $ time perf script tools/perf/scripts/python/syscall-counts-by-pid.py perf Install the python-audit package to get syscall names. For example: # apt-get install python3-audit (Ubuntu) # yum install python3-audit (Fedora) etc. Press control+C to stop and show the summary Warning: 1 out of order events recorded. syscall events for perf: comm [pid]/syscalls count --------------------------------------- ---------- perf [3886080] 1 538989 16 32 203 17 3 2 257 1 204 1 15 1 0 1 perf [3886082] 7 1 real 0m3.852s user 0m3.512s sys 0m0.336s $ time python3 tools/perf/python/syscall-counts-by-pid.py perf Warning: 1 out of order events recorded. syscall events for perf: comm [pid]/syscalls count --------------------------------------- ----------- perf [3886080] write 538989 ioctl 32 sched_setaffinity 17 close 2 openat 1 sched_getaffinity 1 rt_sigreturn 1 read 1 perf [3886082] poll 1 real 0m1.011s user 0m0.963s sys 0m0.048s ```
--- tools/perf/python/syscall-counts-by-pid.py | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 tools/perf/python/syscall-counts-by-pid.py
tools/perf/python/syscall-counts-by-pid.py new file +57 -0
@@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Displays system-wide system call totals, broken down by syscall. +# If a [comm] arg is specified, only syscalls called by [comm] are displayed. + +import argparse +from collections import defaultdict +import perf + +syscalls = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) +for_comm = None +session = None + + +def print_syscall_totals(): + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'comm [pid]/syscalls':<40} {'count':>10}") + print("---------------------------------------- -----------") + + comm_keys = syscalls.keys() + for comm in comm_keys: + pid_keys = syscalls[comm].keys() + for pid in pid_keys: + print(f"\n{comm} [{pid}]") + id_keys = syscalls[comm][pid].keys() + for id, val in sorted(syscalls[comm][pid].items(), + key=lambda kv: (kv[1], kv[0]), reverse=True): + print(f" {perf.syscall_name(id):<38} {val:>10}") + + +def process_event(sample): + event_name = str(sample.evsel) + if event_name == "evsel(raw_syscalls:sys_enter)": + id = sample.id + elif event_name.startswith("evsel(syscalls:sys_enter_"): + id = sample.__syscall_nr + else: + return + pid = sample.sample_pid + comm = session.process(pid).comm() + if for_comm and comm != for_comm: + return + syscalls[comm][pid][id] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("comm", nargs="?", help="Only report syscalls for comm") + args = ap.parse_args() + for_comm = args.comm + session = perf.session(perf.data("perf.data"), sample=process_event) + session.process_events() + print_syscall_totals()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 28/58] perf syscall-counts-by-pid: Port syscall-counts-by-pid to use python module
Sashiko AI review found 5 potential issue(s): - [High] Unhandled `TypeError` when looking up untracked PIDs crashes the script. - [High] Python f-string formatting crashes with a `TypeError` for…
Sashiko AI review found 5 potential issue(s):
- [High] Unhandled TypeError when looking up untracked PIDs crashes the script.
- [High] Python f-string formatting crashes with a TypeError for unknown syscall IDs.
- [High] The script ignores input file arguments and hardcodes 'perf.data', breaking -i/--input support.
- [Medium] Feature regression: Removed the ability to filter syscall counts by Process ID.
- [Low] Unused variable id_keys in the summary printing loop.
--
commit 4158f24426deccec5a61785f2dd5ad7903a4ad84
Author: Ian Rogers <[email protected]>
perf syscall-counts-by-pid: Port syscall-counts-by-pid to use python module
This commit rewrites the syscall-counts-by-pid script to use the perf Python module, significantly improving execution performance. It replaces the legacy event processing and filtering logic.
Quoted 6 lines
tools/perf/python/syscall-counts-by-pid.py new file +0
-0@@ -0,0 +1,57 @@
[ ... ]
Quoted 1 line
+def
print_syscall_totals():
[ ... ]
tools/perf/python/syscall-counts-by-pid.py line 29
Quoted 6 lines
tools/perf/python/syscall-counts-by-pid.py new file +6
-0+ comm_keys = syscalls.keys() + for comm in comm_keys: + pid_keys = syscalls[comm].keys() + for pid in pid_keys: + print(f"\n{comm} [{pid}]") + id_keys = syscalls[comm][pid].keys()
This isn't a bug, but is the id_keys variable needed? It appears to be unused.
tools/perf/python/syscall-counts-by-pid.py line 32
Quoted 3 lines
tools/perf/python/syscall-counts-by-pid.py new file +3
-0+ for id, val in sorted(syscalls[comm][pid].items(), + key=lambda kv: (kv[1], kv[0]), reverse=True): + print(f" {perf.syscall_name(id):<38} {val:>10}")
Will this crash with a TypeError if an unknown syscall ID is encountered?
If perf.syscall_name(id) returns None for an unmapped syscall, passing None to the string format specifier (:<38) will raise a TypeError.
[ ... ]
Quoted 2 lines
tools/perf/python/syscall-counts-by-pid.py new file +2
-0+def process_event(sample): + event_name = str(sample.evsel)
[ ... ]
tools/perf/python/syscall-counts-by-pid.py line 44
Quoted 2 lines
tools/perf/python/syscall-counts-by-pid.py new file +2
-0+ pid = sample.sample_pid + comm = session.process(pid).comm()
Is a try...except block needed here to handle untracked PIDs?
If the underlying C extension fails to find the thread for the PID (which can happen for short-lived processes or dropped events), it raises a Python TypeError.
Without handling this exception, encountering a single untracked PID could abruptly terminate the entire analysis session.
[ ... ]
tools/perf/python/syscall-counts-by-pid.py line 52
Quoted 3 lines
tools/perf/python/syscall-counts-by-pid.py new file +3
-0+if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("comm", nargs="?", help="Only report syscalls for comm")
Does this remove the ability to filter syscall counts by Process ID?
The original legacy script allowed users to filter by Process ID by checking if the argument was parseable as an integer.
With this change, passing a PID will interpret the input as a literal string command name, silently failing to filter.
tools/perf/python/syscall-counts-by-pid.py line 55
Quoted 3 lines
tools/perf/python/syscall-counts-by-pid.py new file +3
-0+ args = ap.parse_args() + for_comm = args.comm + session = perf.session(perf.data("perf.data"), sample=process_event)
Does hardcoding perf.data break the ability to use custom input files?
Because the script's argparse configuration does not accept an -i override, it appears it will completely ignore custom input files passed via perf script (e.g., perf script -i mytrace.data) and always process perf.data from the current directory.
Quoted 2 lines
tools/perf/python/syscall-counts-by-pid.py new file +2
-0+ session.process_events() + print_syscall_totals()
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=28
Ian Rogers
reviewerPATCH 29/58 perf futex-contention: Port futex-contention to use python module
In reply to Ian RogersRewrite tools/perf/scripts/python/futex-contention.py to use the python module and various style changes. By avoiding the overheads in the perf script execution the performance improves by more than 3.2x as shown in the following (with PYTHON_PATH and PERF_EXEC_PATH set as necessary):
``` $ perf record -e syscalls:sys_*_futex -a sleep 1 ... $ time perf script tools/perf/scripts/python/futex-contention.py Install the python-audit package to get syscall names. For example: # apt-get install python3-audit (Ubuntu) # yum install python3-audit (Fedora) etc. Press control+C to stop and show the summary aaa/4[2435653] lock 7f76b380c878 contended 1 times, 1099 avg ns [max: 1099 ns, min 1099 ns] ... real 0m1.007s user 0m0.935s sys 0m0.072s $ time python3 tools/perf/python/futex-contention.py ... real 0m0.314s user 0m0.259s sys 0m0.056s ```
--- tools/perf/python/futex-contention.py | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 tools/perf/python/futex-contention.py
tools/perf/python/futex-contention.py new file +54 -0
@@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Measures futex contention. +from collections import defaultdict +from typing import Dict +import perf + +process_names: Dict[int, str] = {} +start_times: Dict[int, [int, int]] = {} +session = None +durations: Dict[[int, int], list] = defaultdict(list) + +FUTEX_WAIT = 0 +FUTEX_WAKE = 1 +FUTEX_PRIVATE_FLAG = 128 +FUTEX_CLOCK_REALTIME = 256 +FUTEX_CMD_MASK = ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) + + +def process_event(sample): + def handle_start(tid: int, uaddr: int, op: int, start_time: int) -> None: + if (op & FUTEX_CMD_MASK) != FUTEX_WAIT: + return + if tid not in process_names: + try: + process = session.process(tid) + except Exception: + return + process_names[tid] = process.comm() + start_times[tid] = (uaddr, start_time) + + def handle_end(tid: int, end_time: int) -> None: + if tid not in start_times: + return + (uaddr, start_time) = start_times[tid] + del start_times[tid] + durations[(tid, uaddr)].append(end_time - start_time) + event_name = str(sample.evsel) + if event_name == "evsel(syscalls:sys_enter_futex)": + handle_start(sample.sample_tid, sample.uaddr, + sample.op, sample.sample_time) + elif event_name == "evsel(syscalls:sys_exit_futex)": + handle_end(sample.sample_tid, sample.sample_time) + + +if __name__ == "__main__": + session = perf.session(perf.data("perf.data"), sample=process_event) + session.process_events() + for ((tid, uaddr), vals) in sorted(durations.items()): + avg = sum(vals) / len(vals) + max_val = max(vals) + min_val = min(vals) + print(f"{process_names[tid]}[{tid}] lock {uaddr:x} contended {len(vals)} " + f"times, {avg} avg ns [max: {max_val} ns, min {min_val} ns]")
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 29/58] perf futex-contention: Port futex-contention to use python module
Sashiko AI review found 3 potential issue(s): - [High] Fatal TypeError due to invalid type hinting syntax. - [Medium] Unbounded memory growth leading to OOM risks on large trace files. - [Medium]…
Sashiko AI review found 3 potential issue(s):
- [High] Fatal TypeError due to invalid type hinting syntax.
- [Medium] Unbounded memory growth leading to OOM risks on large trace files.
- [Medium] Hardcoded input file drops support for custom trace files. --
commit 1b6a7eb3db688a2d738e2a6cae094deea81591b4
Author: Ian Rogers <[email protected]>
perf futex-contention: Port futex-contention to use python module
This commit rewrites tools/perf/scripts/python/futex-contention.py to use the python module. It aims to improve performance by avoiding the overheads in the perf script execution.
Quoted 7 lines
tools/perf/python/futex-contention.py new file +1
-0@@ -0,0 +1,54 @@ +#!/usr/bin/env python3
tools/perf/python/futex-contention.py line 11
Quoted 10 lines
tools/perf/python/futex-contention.py new file +10
-0+# SPDX-License-Identifier: GPL-2.0 +# Measures futex contention. +from collections import defaultdict +from typing import Dict +import perf + +process_names: Dict[int, str] = {} +start_times: Dict[int, [int, int]] = {} +session = None +durations: Dict[[int, int], list] = defaultdict(list)
Will this raise a TypeError at module import time? typing.Dict requires types as arguments, not list instances. Should this use Tuple[int, int] instead?
[ ... ]
tools/perf/python/futex-contention.py line 37
Quoted 6 lines
tools/perf/python/futex-contention.py new file +6
-0+ def handle_end(tid: int, end_time: int) -> None: + if tid not in start_times: + return + (uaddr, start_time) = start_times[tid] + del start_times[tid] + durations[(tid, uaddr)].append(end_time - start_time)
Does storing every single duration in a list risk unbounded memory growth and potential Out-Of-Memory crashes on large traces with high-frequency futex workloads?
The original perf script based utility function maintained a running O(1) memory aggregate (min, max, avg, count) for each lock.
[ ... ]
tools/perf/python/futex-contention.py line 47
Quoted 2 lines
tools/perf/python/futex-contention.py new file +2
-0+if __name__ == "__main__": + session = perf.session(perf.data("perf.data"), sample=process_event)
Does this hardcode the input file to perf.data in the current working directory?
Unlike other ported scripts that use argparse and sys.argv, this script drops support for custom trace files. This breaks compatibility with workflows where users specify custom trace locations (e.g., perf script -i my_trace.data).
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=29
Ian Rogers
reviewerPATCH 30/58 perf flamegraph: Port flamegraph to use python module
In reply to Ian RogersAdd a port of the flamegraph script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
--- tools/perf/python/flamegraph.py | 242 ++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100755 tools/perf/python/flamegraph.py
tools/perf/python/flamegraph.py new file +242 -0
@@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +flamegraph.py - create flame graphs from perf samples using perf python module +""" + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import urllib.request +from typing import Dict, Optional, Union +import perf + +MINIMAL_HTML = """<head> + <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/d3-flamegraph.css"> +</head> +<body> + <div id="chart"></div> + <script type="text/javascript" src="https://d3js.org/d3.v7.js"></script> + <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/[email protected]/dist/d3-flamegraph.min.js"></script> + <script type="text/javascript"> + const stacks = [/** @flamegraph_json **/]; + // Note, options is unused. + const options = [/** @options_json **/]; + + var chart = flamegraph(); + d3.select("#chart") + .datum(stacks[0]) + .call(chart); + </script> +</body> +""" + +class Node: + """A node in the flame graph tree.""" + def __init__(self, name: str, libtype: str): + self.name = name + self.libtype = libtype + self.value: int = 0 + self.children: list[Node] = [] + + def to_json(self) -> Dict[str, Union[str, int, list[Dict]]]: + """Convert the node to a JSON-serializable dictionary.""" + return { + "n": self.name, + "l": self.libtype, + "v": self.value, + "c": [x.to_json() for x in self.children] + } + + +class FlameGraphCLI: + """Command-line interface for generating flame graphs.""" + def __init__(self, args): + self.args = args + self.stack = Node("all", "root") + self.session = None + + @staticmethod + def get_libtype_from_dso(dso: Optional[str]) -> str: + """Determine the library type from the DSO name.""" + if dso and (dso == "[kernel.kallsyms]" or dso.endswith("/vmlinux") or dso == "[kernel]"): + return "kernel" + return "" + + @staticmethod + def find_or_create_node(node: Node, name: str, libtype: str) -> Node: + """Find a child node with the given name or create a new one.""" + for child in node.children: + if child.name == name: + return child + child = Node(name, libtype) + node.children.append(child) + return child + + def process_event(self, sample) -> None: + """Process a single perf sample event.""" + if self.args.event_name and str(sample.evsel) != self.args.event_name: + return + + pid = sample.sample_pid + dso_type = "" + try: + thread = self.session.process(sample.sample_tid) + comm = thread.comm() + # Try to get libtype from thread's main executable + # This is a bit of a hack as pyrf_thread doesn't expose maps easily + except Exception: + comm = "[unknown]" + + if pid == 0: + comm = "swapper" + dso_type = "kernel" + else: + comm = f"{comm} ({pid})" + + node = self.find_or_create_node(self.stack, comm, dso_type) + + callchain = sample.callchain + if callchain: + # We want to traverse from root to leaf. + # perf callchain iterator gives leaf to root. + # We collect them and reverse. + frames = list(callchain) + for entry in reversed(frames): + name = entry.symbol or "[unknown]" + libtype = self.get_libtype_from_dso(entry.dso) + node = self.find_or_create_node(node, name, libtype) + node.value += 1 + + def get_report_header(self) -> str: + """Get the header from the perf report.""" + try: + input_file = self.args.input or "perf.data" + output = subprocess.check_output(["perf", "report", "--header-only", "-i", input_file]) + result = output.decode("utf-8") + if self.args.event_name: + result += "\nFocused event: " + self.args.event_name + return result + except Exception: + return "" + + def run(self) -> None: + """Run the flame graph generation.""" + input_file = self.args.input or "perf.data" + if not os.path.exists(input_file): + print(f"Error: {input_file} not found. (try 'perf record' first)", file=sys.stderr) + sys.exit(1) + + try: + self.session = perf.session(perf.data(input_file), + sample=self.process_event) + except Exception as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + self.session.process_events() + + stacks_json = json.dumps(self.stack, default=lambda x: x.to_json()) + + if self.args.format == "html": + report_header = self.get_report_header() + options = { + "colorscheme": self.args.colorscheme, + "context": report_header + } + options_json = json.dumps(options) + + template = self.args.template + template_md5sum = None + output_str = None + if not os.path.isfile(template): + if not self.args.allow_download: + print(f"Warning: Flame Graph template '{template}' does not exist.", + file=sys.stderr) + template = ( + "https://cdn.jsdelivr.net/npm/[email protected]/dist/templates/" + "d3-flamegraph-base.html" + ) + template_md5sum = "143e0d06ba69b8370b9848dcd6ae3f36" + if not sys.stdin.isatty(): + print("Non-interactive environment detected, skipping download.") + template = None + else: + s = None + while s not in ["y", "n"]: + s = input("Do you wish to download a template from cdn.jsdelivr.net? " + + "(this warning can be suppressed with --allow-download) [yn] " + ).lower() + if s == "n": + template = None + template_md5sum = None + + use_minimal = False + try: + if not template: + use_minimal = True + elif template.startswith("http"): + with urllib.request.urlopen(template) as url_template: + output_str = "".join([l.decode("utf-8") for l in url_template.readlines()]) + else: + with open(template, "r", encoding="utf-8") as f: + output_str = f.read() + except Exception as err: + print(f"Error reading template {template}: {err}\n", file=sys.stderr) + use_minimal = True + + if use_minimal: + print("Using internal minimal HTML that refers to d3's web site. JavaScript " + + "loaded this way from a local file is typically blocked unless your " + + "browser has relaxed permissions.") + output_str = MINIMAL_HTML + + elif template_md5sum: + download_md5sum = hashlib.md5(output_str.encode("utf-8")).hexdigest() + if download_md5sum != template_md5sum: + s = None + while s not in ["y", "n"]: + s = input(f"""Unexpected template md5sum. +{download_md5sum} != {template_md5sum}, for: +{output_str} +continue?[yn] """).lower() + if s == "n": + sys.exit(1) + + output_str = output_str.replace("/** @options_json **/", options_json) + output_str = output_str.replace("/** @flamegraph_json **/", stacks_json) + output_fn = self.args.output or "flamegraph.html" + else: + output_str = stacks_json + output_fn = self.args.output or "stacks.json" + + if output_fn == "-": + sys.stdout.write(output_str) + else: + print(f"dumping data to {output_fn}") + with open(output_fn, "w", encoding="utf-8") as out: + out.write(output_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Create flame graphs using perf python module.") + parser.add_argument("-f", "--format", default="html", choices=["json", "html"], + help="output file format") + parser.add_argument("-o", "--output", help="output file name") + parser.add_argument("--template", + default="/usr/share/d3-flame-graph/d3-flamegraph-base.html", + help="path to flame graph HTML template") + parser.add_argument("--colorscheme", default="blue-green", + help="flame graph color scheme", choices=["blue-green", "orange"]) + parser.add_argument("-i", "--input", help="input perf.data file") + parser.add_argument("--allow-download", default=False, action="store_true", + help="allow unprompted downloading of HTML template") + parser.add_argument("-e", "--event", default="", dest="event_name", type=str, + help="specify the event to generate flamegraph for") + + cli_args = parser.parse_args() + cli = FlameGraphCLI(cli_args) + cli.run()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 30/58] perf flamegraph: Port flamegraph to use python module
Sashiko AI review found 9 potential issue(s): - [Medium] Python 3.9+ type hinting syntax `list[Node]` and `list[Dict]` raises TypeError on older Python versions. - [Medium] The `--allow-download`…
Sashiko AI review found 9 potential issue(s):
- [Medium] Python 3.9+ type hinting syntax list[Node] and list[Dict] raises TypeError on older Python versions.
- [Medium] The --allow-download flag logic is broken and inadvertently prevents downloading the template.
- [Medium] Faulty handling of custom HTTP URLs passed to --template.
- [Medium] Informational minimal HTML fallback message is printed to stdout, corrupting data streams.
- [Medium] Unsanitized JSON data injected into HTML <script> tag allows Local HTML Injection / XSS.
- [Medium] Missing fallback logic for samples without callchains.
- [Medium] Terminal escape sequence injection during MD5 checksum failure prompt.
- [Medium] Algorithmic complexity bottleneck in find_or_create_node causes O(N^2) performance degradation.
- [Low] Weak cryptographic hash algorithm (MD5) used for template integrity verification.
--
commit fddc7fd3e79b699ace56746bc19782f556d9736d
Author: Ian Rogers <[email protected]>
perf flamegraph: Port flamegraph to use python module
Add a port of the flamegraph script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
Quoted 5 lines
tools/perf/python/flamegraph.py new file +0
-0
[ ... ]
tools/perf/python/flamegraph.py line 43
Quoted 7 lines
tools/perf/python/flamegraph.py new file +7
-0+class Node: + """A node in the flame graph tree.""" + def __init__(self, name: str, libtype: str): + self.name = name + self.libtype = libtype + self.value: int = 0 + self.children: list[Node] = []
Will this script raise a TypeError on older Python versions? Using list[Node] for type hinting relies on PEP 585, which requires Python 3.9+. Could this use typing.List instead to maintain compatibility with older environments?
tools/perf/python/flamegraph.py
Quoted 2 lines
tools/perf/python/flamegraph.py new file +2
-0+ + def to_json(self) -> Dict[str, Union[str, int, list[Dict]]]:
Similarly, does using list[Dict] here cause compatibility issues on versions prior to Python 3.9?
[ ... ]
tools/perf/python/flamegraph.py line 77
Quoted 9 lines
tools/perf/python/flamegraph.py new file +9
-0+ @staticmethod + def find_or_create_node(node: Node, name: str, libtype: str) -> Node: + """Find a child node with the given name or create a new one.""" + for child in node.children: + if child.name == name: + return child + child = Node(name, libtype) + node.children.append(child) + return child
Could this linear search loop cause severe performance degradation? For exceptionally wide call graphs with many unique symbols at the same depth, iterating over node.children for every frame could create an O(N^2) bottleneck.
[ ... ]
Quoted 1 line
+ def
process_event(self, sample)-> None:
[ ... ]
tools/perf/python/flamegraph.py line 112
Quoted 11 lines
tools/perf/python/flamegraph.py new file +11
-0+ callchain = sample.callchain + if callchain: + # We want to traverse from root to leaf. + # perf callchain iterator gives leaf to root. + # We collect them and reverse. + frames = list(callchain) + for entry in reversed(frames): + name = entry.symbol or "[unknown]" + libtype = self.get_libtype_from_dso(entry.dso) + node = self.find_or_create_node(node, name, libtype) + node.value += 1
Does this skip adding sample details if the profile was recorded without callchains?
If sample.callchain is empty, the script skips the frame processing loop entirely and only increments the root process node, which might result in flamegraphs that only display process names. Could it fall back to the instruction pointer or symbol?
[ ... ]
Quoted 1 line
+ def run(self) -> None:
[ ... ]
tools/perf/python/flamegraph.py line 162
Quoted 11 lines
tools/perf/python/flamegraph.py new file +11
-0+ template = self.args.template + template_md5sum = None + output_str = None + if not os.path.isfile(template): + if not self.args.allow_download: + print(f"Warning: Flame Graph template '{template}' does not exist.", + file=sys.stderr) + template = ( + "https://cdn.jsdelivr.net/npm/[email protected]/dist/templates/" + "d3-flamegraph-base.html" + )
Does this logic inadvertently prevent the template from downloading when --allow-download is passed?
If self.args.allow_download is true, the script skips updating the template variable to the CDN URL, keeping it as the non-existent local path. This causes the subsequent open() to fail and fall back to minimal HTML.
Also, can this overwrite a custom HTTP URL provided via the --template argument?
Since os.path.isfile() evaluates to false for HTTP URLs, providing a custom URL without --allow-download causes the script to warn that the template does not exist and silently overwrite the user's URL with the default cdn.jsdelivr.net URL.
[ ... ]
tools/perf/python/flamegraph.py line 195
Quoted 5 lines
tools/perf/python/flamegraph.py new file +5
-0+ if use_minimal: + print("Using internal minimal HTML that refers to d3's web site. JavaScript " + + "loaded this way from a local file is typically blocked unless your " + + "browser has relaxed permissions.") + output_str = MINIMAL_HTML
Will this print statement corrupt the output stream if the user specifies -o - to output the data to stdout? Writing informational warnings to standard output instead of standard error might cause downstream tools or browsers to fail when parsing the resulting HTML or JSON.
tools/perf/python/flamegraph.py line 198
Quoted 3 lines
tools/perf/python/flamegraph.py new file +3
-0+ + elif template_md5sum: + download_md5sum = hashlib.md5(output_str.encode("utf-8")).hexdigest()
Is MD5 robust enough for template integrity verification? Since MD5 is vulnerable to collision attacks, might it allow a maliciously modified template to produce the expected hash?
tools/perf/python/flamegraph.py line 205
Quoted 7 lines
tools/perf/python/flamegraph.py new file +7
-0+ if download_md5sum != template_md5sum: + s = None + while s not in ["y", "n"]: + s = input(f"""Unexpected template md5sum. +{download_md5sum} != {template_md5sum}, for: +{output_str} +continue?[yn] """).lower()
Can printing the unverified output_str directly to the terminal inject malicious ANSI escape sequences? If the downloaded template from an untrusted source fails the checksum, embedding the raw payload into the input() prompt could manipulate the user's terminal emulator.
tools/perf/python/flamegraph.py line 210
Quoted 5 lines
tools/perf/python/flamegraph.py new file +5
-0+ if s == "n": + sys.exit(1) + + output_str = output_str.replace("/** @options_json **/", options_json) + output_str = output_str.replace("/** @flamegraph_json **/", stacks_json)
Does embedding the JSON directly into the HTML script block allow for cross-site scripting or HTML injection?
Since json.dumps() does not escape HTML special characters like < or /, could a maliciously crafted process or thread name prematurely terminate the script block and execute arbitrary JavaScript?
Quoted 1 line
+
output_fn= self.args.output or "flamegraph.html"
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=30
Ian Rogers
reviewerPATCH 31/58 perf gecko: Port gecko to use python module
In reply to Ian RogersAdd a port of the gecko script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
--- tools/perf/python/gecko.py | 372 +++++++++++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100755 tools/perf/python/gecko.py
tools/perf/python/gecko.py new file +372 -0
@@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +gecko.py - Convert perf record output to Firefox's gecko profile format +""" + +import argparse +import json +import os +import subprocess +import sys +import threading +import urllib.parse +import webbrowser +from dataclasses import dataclass, field +from http.server import HTTPServer, SimpleHTTPRequestHandler +from typing import Dict, List, NamedTuple, Optional, Tuple + +import perf + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 +class Frame(NamedTuple): + """A single stack frame in the gecko profile format.""" + string_id: int + relevantForJS: bool + innerWindowID: int + implementation: None + optimizations: None + line: None + column: None + category: int + subcategory: Optional[int] + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 +class Stack(NamedTuple): + """A single stack in the gecko profile format.""" + prefix_id: Optional[int] + frame_id: int + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 +class Sample(NamedTuple): + """A single sample in the gecko profile format.""" + stack_id: Optional[int] + time_ms: float + responsiveness: int + + +@dataclass +class Tables: + """Interned tables for the gecko profile format.""" + frame_table: List[Frame] = field(default_factory=list) + string_table: List[str] = field(default_factory=list) + string_map: Dict[str, int] = field(default_factory=dict) + stack_table: List[Stack] = field(default_factory=list) + stack_map: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frame_map: Dict[str, int] = field(default_factory=dict) + + +@dataclass +class Thread: + """A builder for a profile of the thread.""" + comm: str + pid: int + tid: int + user_category: int + kernel_category: int + samples: List[Sample] = field(default_factory=list) + tables: Tables = field(default_factory=Tables) + + def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: + """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" + key = (prefix_id, frame_id) + stack_id = self.tables.stack_map.get(key) + if stack_id is None: + stack_id = len(self.tables.stack_table) + self.tables.stack_table.append(Stack(prefix_id=prefix_id, frame_id=frame_id)) + self.tables.stack_map[key] = stack_id + return stack_id + + def _intern_string(self, string: str) -> int: + """Gets a matching string, or saves the new string. Returns a String ID.""" + string_id = self.tables.string_map.get(string) + if string_id is not None: + return string_id + string_id = len(self.tables.string_table) + self.tables.string_table.append(string) + self.tables.string_map[string] = string_id + return string_id + + def _intern_frame(self, frame_str: str) -> int: + """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" + frame_id = self.tables.frame_map.get(frame_str) + if frame_id is not None: + return frame_id + frame_id = len(self.tables.frame_table) + self.tables.frame_map[frame_str] = frame_id + string_id = self._intern_string(frame_str) + + category = self.user_category + if (frame_str.find('kallsyms') != -1 or + frame_str.find('/vmlinux') != -1 or + frame_str.endswith('.ko)')): + category = self.kernel_category + + self.tables.frame_table.append(Frame( + string_id=string_id, + relevantForJS=False, + innerWindowID=0, + implementation=None, + optimizations=None, + line=None, + column=None, + category=category, + subcategory=None, + )) + return frame_id + + def add_sample(self, comm: str, stack: List[str], time_ms: float) -> None: + """Add a timestamped stack trace sample to the thread builder.""" + if self.comm != comm: + self.comm = comm + + prefix_stack_id: Optional[int] = None + for frame in stack: + frame_id = self._intern_frame(frame) + prefix_stack_id = self._intern_stack(frame_id, prefix_stack_id) + + if prefix_stack_id is not None: + self.samples.append(Sample(stack_id=prefix_stack_id, + time_ms=time_ms, + responsiveness=0)) + + def to_json_dict(self) -> Dict: + """Converts current Thread to GeckoThread JSON format.""" + return { + "tid": self.tid, + "pid": self.pid, + "name": self.comm, + "markers": { + "schema": { + "name": 0, + "startTime": 1, + "endTime": 2, + "phase": 3, + "category": 4, + "data": 5, + }, + "data": [], + }, + "samples": { + "schema": { + "stack": 0, + "time": 1, + "responsiveness": 2, + }, + "data": self.samples + }, + "frameTable": { + "schema": { + "location": 0, + "relevantForJS": 1, + "innerWindowID": 2, + "implementation": 3, + "optimizations": 4, + "line": 5, + "column": 6, + "category": 7, + "subcategory": 8, + }, + "data": self.tables.frame_table, + }, + "stackTable": { + "schema": { + "prefix": 0, + "frame": 1, + }, + "data": self.tables.stack_table, + }, + "stringTable": self.tables.string_table, + "registerTime": 0, + "unregisterTime": None, + "processType": "default", + } + + +class CORSRequestHandler(SimpleHTTPRequestHandler): + """Enable CORS for requests from profiler.firefox.com.""" + def end_headers(self): + self.send_header('Access-Control-Allow-Origin', 'https://profiler.firefox.com') + super().end_headers() + + +@dataclass +class CategoryData: + """Category configuration for the gecko profile.""" + user_index: int = 0 + kernel_index: int = 1 + categories: List[Dict] = field(default_factory=list) + + +class GeckoCLI: + """Command-line interface for converting perf data to Gecko format.""" + def __init__(self, args): + self.args = args + self.tid_to_thread: Dict[int, Thread] = {} + self.start_time_ms: Optional[float] = None + self.session = None + self.product = subprocess.check_output(['uname', '-op']).decode().strip() + self.cat_data = CategoryData( + categories=[ + { + "name": 'User', + "color": args.user_color, + "subcategories": ['Other'] + }, + { + "name": 'Kernel', + "color": args.kernel_color, + "subcategories": ['Other'] + }, + ] + ) + + def process_event(self, sample) -> None: + """Process a single perf sample event.""" + if self.args.event_name and str(sample.evsel) != self.args.event_name: + return + + # sample_time is in nanoseconds. Gecko wants milliseconds. + time_ms = sample.sample_time / 1000000.0 + pid = sample.sample_pid + tid = sample.sample_tid + + if self.start_time_ms is None: + self.start_time_ms = time_ms + + try: + thread_info = self.session.process(tid) + comm = thread_info.comm() + except Exception: + comm = "[unknown]" + + stack = [] + callchain = sample.callchain + if callchain: + for entry in callchain: + symbol = entry.symbol or "[unknown]" + dso = entry.dso or "[unknown]" + stack.append(f"{symbol} (in {dso})") + # Reverse because Gecko wants root first. + stack.reverse() + else: + # Fallback if no callchain is present + try: + # If the perf module exposes symbol/dso directly on sample + # when callchain is missing, we use them. + symbol = getattr(sample, 'symbol', '[unknown]') + dso = getattr(sample, 'dso', '[unknown]') + stack.append(f"{symbol} (in {dso})") + except AttributeError: + stack.append("[unknown] (in [unknown])") + + thread = self.tid_to_thread.get(tid) + if thread is None: + thread = Thread(comm=comm, pid=pid, tid=tid, + user_category=self.cat_data.user_index, + kernel_category=self.cat_data.kernel_index) + self.tid_to_thread[tid] = thread + thread.add_sample(comm=comm, stack=stack, time_ms=time_ms) + + def run(self) -> None: + """Run the conversion process.""" + input_file = self.args.input or "perf.data" + if not os.path.exists(input_file): + print(f"Error: {input_file} not found.", file=sys.stderr) + sys.exit(1) + + try: + self.session = perf.session(perf.data(input_file), sample=self.process_event) + except Exception as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + self.session.process_events() + + threads = [t.to_json_dict() for t in self.tid_to_thread.values()] + + gecko_profile = { + "meta": { + "interval": 1, + "processType": 0, + "product": self.product, + "stackwalk": 1, + "debug": 0, + "gcpoison": 0, + "asyncstack": 1, + "startTime": self.start_time_ms, + "shutdownTime": None, + "version": 24, + "presymbolicated": True, + "categories": self.cat_data.categories, + "markerSchema": [], + }, + "libs": [], + "threads": threads, + "processes": [], + "pausedRanges": [], + } + + output_file = self.args.save_only + if output_file is None: + output_file = 'gecko_profile.json' + self._write_and_launch(gecko_profile, output_file) + else: + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(gecko_profile, f, indent=2) + + def _write_and_launch(self, profile: Dict, filename: str) -> None: + """Write the profile to a file and launch the Firefox profiler.""" + print("Starting Firefox Profiler on your default browser...") + with open(filename, 'w', encoding='utf-8') as f: + json.dump(profile, f, indent=2) + + # Start HTTP server in a daemon thread + def start_server(): + server_address = ('', 8000) + httpd = HTTPServer(server_address, CORSRequestHandler) + httpd.serve_forever() + + thread = threading.Thread(target=start_server, daemon=True) + thread.start() + + # Open the browser + safe_string = urllib.parse.quote_plus(f'http://localhost:8000/{filename}') + url = f'https://profiler.firefox.com/from-url/{safe_string}' + webbrowser.open(url) + + print(f'[ perf gecko: Captured and wrote into {filename} ]') + print("Press Ctrl+C to stop the local server.") + try: + # Keep the main thread alive so the daemon thread can serve requests + while True: + threading.Event().wait(1) + except KeyboardInterrupt: + print("\nStopping server...") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convert perf.data to Firefox's Gecko Profile format" + ) + parser.add_argument('--user-color', default='yellow', + help='Color for the User category', + choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', + 'grey', 'magenta']) + parser.add_argument('--kernel-color', default='orange', + help='Color for the Kernel category', + choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', + 'grey', 'magenta']) + parser.add_argument('--save-only', + help='Save the output to a file instead of opening Firefox\'s profiler') + parser.add_argument("-i", "--input", help="input perf.data file") + parser.add_argument("-e", "--event", default="", dest="event_name", type=str, + help="specify the event to generate gecko profile for") + + cli_args = parser.parse_args() + cli = GeckoCLI(cli_args) + cli.run()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 31/58] perf gecko: Port gecko to use python module
Sashiko AI review found 5 potential issue(s): - [High] The HTTP server binds to all network interfaces, exposing the current working directory to the local network. - [Medium] Unresolved symbols are…
Sashiko AI review found 5 potential issue(s):
- [High] The HTTP server binds to all network interfaces, exposing the current working directory to the local network.
- [Medium] Unresolved symbols are formatted as literal 'None' strings instead of the intended '[unknown]'.
- [Medium] The script uses a hardcoded port (8000) without handling 'Address already in use' errors, causing silent hangs.
- [Medium] Using uname -op crashes the script entirely on non-Linux platforms (e.g., macOS).
- [Low] Race condition between HTTP server initialization and browser request.
--
commit dc4b7f657c6a2aa78c3372c06ef2745b8abfea67
Author: Ian Rogers <[email protected]>
perf gecko: Port gecko to use python module
This commit adds a port of the gecko script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries.
Quoted 5 lines
tools/perf/python/gecko.py new file +0
-0
[ ... ]
Quoted 8 lines
class GeckoCLI:
"""Command-line interface for converting perf data to Gecko format.""" def__init__(self, args):
self.args = args
self.tid_to_thread: Dict[int, Thread] = {} self.start_time_ms: Optional[float] = None
self.session = None
self.product =subprocess.check_output(['uname', '-op']).decode().strip()
Can this crash on non-Linux platforms like macOS or BSD? The -o and -p options are GNU coreutils extensions and might raise a subprocess.CalledProcessError on systems where they are invalid.
[ ... ]
Quoted 8 lines
else:
# Fallback if no callchain is present
try:
# If the perf module exposes symbol/dso directly on sample # when callchain is missing, we use them. symbol =getattr(sample, 'symbol', '[unknown]')dso =getattr(sample, 'dso', '[unknown]')stack.append(f"{symbol} (in {dso})")
If the C extension explicitly returns Python's None for unresolved symbols rather than raising an AttributeError, will getattr successfully return None here? This would cause the f-string to evaluate to the literal string "None (in None)" rather than the intended [unknown] fallback.
[ ... ]
Quoted 9 lines
def
_write_and_launch(self, profile: Dict, filename: str)-> None:
"""Write the profile to a file and launch the Firefox profiler."""print("Starting Firefox Profiler on your default browser...")withopen(filename, 'w', encoding='utf-8')as f:
json.dump(profile, f, indent=2)# Start HTTP server in a daemon thread
defstart_server():
server_address= ('', 8000)
Does this configuration expose the current working directory to the entire local network? By binding to an empty string, it listens on 0.0.0.0 (all available interfaces) rather than restricting it securely to 127.0.0.1.
Quoted 1 line
httpd =
HTTPServer(server_address, CORSRequestHandler)
What happens if port 8000 is already in use by another process? It looks like HTTPServer will raise an OSError, causing the daemon thread to crash silently while leaving the main thread trapped in an infinite while True loop.
Quoted 9 lines
httpd.serve_forever()thread =
threading.Thread(target=start_server, daemon=True)thread.start()# Open the browser
safe_string= urllib.parse.quote_plus(f'http://localhost:8000/{filename}') url = f'https://profiler.firefox.com/from-url/{safe_string}'webbrowser.open(url)
Is there a race condition here where the browser might attempt to fetch the profile before the newly spawned server thread has actually bound to the port and started listening?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=31
Ian Rogers
reviewerPATCH 32/58 perf arm-cs-trace-disasm: Port arm-cs-trace-disasm to use python module
In reply to Ian RogersAdd a port of the arm-cs-trace-disasm script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields. Update the testing to use the ported script.
--- tools/perf/python/arm-cs-trace-disasm.py | 333 ++++++++++++++++++ .../tests/shell/test_arm_coresight_disasm.sh | 10 +- 2 files changed, 339 insertions(+), 4 deletions(-) create mode 100755 tools/perf/python/arm-cs-trace-disasm.py
tools/perf/python/arm-cs-trace-disasm.py new file +333 -0
@@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +arm-cs-trace-disasm.py: ARM CoreSight Trace Dump With Disassember using perf python module +""" + +import os +from os import path +import re +from subprocess import check_output +import argparse +import platform +import sys +from typing import Dict, List, Optional + +# Initialize global dicts and regular expression +DISASM_CACHE: Dict[str, List[str]] = {} +CPU_DATA: Dict[str, int] = {} +DISASM_RE = re.compile(r"^\s*([0-9a-fA-F]+):") +DISASM_FUNC_RE = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") +CACHE_SIZE = 64*1024 +SAMPLE_IDX = -1 + +GLB_SOURCE_FILE_NAME: Optional[str] = None +GLB_LINE_NUMBER: Optional[int] = None +GLB_DSO: Optional[str] = None + +KVER = platform.release() +VMLINUX_PATHS = [ + f"/usr/lib/debug/boot/vmlinux-{KVER}.debug", + f"/usr/lib/debug/lib/modules/{KVER}/vmlinux", + f"/lib/modules/{KVER}/build/vmlinux", + f"/usr/lib/debug/boot/vmlinux-{KVER}", + f"/boot/vmlinux-{KVER}", + "/boot/vmlinux", + "vmlinux" +] + +def default_objdump() -> str: + """Return the default objdump path from perf config or 'objdump'.""" + try: + config = perf.config_get("annotate.objdump") + return str(config) if config else "objdump" + except (AttributeError, TypeError): + return "objdump" + +def find_vmlinux() -> Optional[str]: + """Find the vmlinux file in standard paths.""" + if hasattr(find_vmlinux, "path"): + return getattr(find_vmlinux, "path") + + for v in VMLINUX_PATHS: + if os.access(v, os.R_OK): + setattr(find_vmlinux, "path", v) + return v + setattr(find_vmlinux, "path", None) + return None + +def get_dso_file_path(dso_name: str, dso_build_id: str, vmlinux: Optional[str]) -> str: + """Return the path to the DSO file.""" + if dso_name in ("[kernel.kallsyms]", "vmlinux"): + if vmlinux: + return vmlinux + return find_vmlinux() or dso_name + + if dso_name == "[vdso]": + append = "/vdso" + else: + append = "/elf" + + buildid_dir = os.environ.get('PERF_BUILDID_DIR') + if not buildid_dir: + buildid_dir = os.path.join(os.environ.get('HOME', ''), '.debug') + + dso_path = buildid_dir + "/" + dso_name + "/" + dso_build_id + append + # Replace duplicate slash chars to single slash char + dso_path = dso_path.replace('//', '/', 1) + return dso_path + +def read_disam(dso_fname: str, dso_start: int, start_addr: int, + stop_addr: int, objdump: str) -> List[str]: + """Read disassembly from a DSO file using objdump.""" + addr_range = f"{start_addr}:{stop_addr}:{dso_fname}" + + # Don't let the cache get too big, clear it when it hits max size + if len(DISASM_CACHE) > CACHE_SIZE: + DISASM_CACHE.clear() + + if addr_range in DISASM_CACHE: + disasm_output = DISASM_CACHE[addr_range] + else: + start_addr = start_addr - dso_start + stop_addr = stop_addr - dso_start + disasm = [objdump, "-d", "-z", + f"--start-address={start_addr:#x}", + f"--stop-address={stop_addr:#x}"] + disasm += [dso_fname] + disasm_output = check_output(disasm).decode('utf-8').split('\n') + DISASM_CACHE[addr_range] = disasm_output + + return disasm_output + +def print_disam(dso_fname: str, dso_start: int, start_addr: int, + stop_addr: int, objdump: str) -> None: + """Print disassembly for a given address range.""" + for line in read_disam(dso_fname, dso_start, start_addr, stop_addr, objdump): + m = DISASM_FUNC_RE.search(line) + if m is None: + m = DISASM_RE.search(line) + if m is None: + continue + print(f"\t{line}") + +def print_sample(sample: perf.sample_event) -> None: + """Print sample details.""" + print(f"Sample = {{ cpu: {sample.sample_cpu:04d} addr: {sample.sample_addr:016x} " + f"phys_addr: {sample.sample_phys_addr:016x} ip: {sample.sample_ip:016x} " + f"pid: {sample.sample_pid} tid: {sample.sample_tid} period: {sample.sample_period} " + f"time: {sample.sample_time} index: {SAMPLE_IDX}}}") + +def common_start_str(comm: str, sample: perf.sample_event) -> str: + """Return common start string for sample output.""" + sec = int(sample.sample_time / 1000000000) + ns = sample.sample_time % 1000000000 + cpu = sample.sample_cpu + pid = sample.sample_pid + tid = sample.sample_tid + return f"{comm:>16s} {pid:5u}/{tid:<5u} [{cpu:04d}] {sec:9d}.{ns:09d} " + +def print_srccode(comm: str, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print source code and symbols for a sample.""" + ip = sample.sample_ip + if symbol == "[unknown]": + start_str = common_start_str(comm, sample) + f"{ip:x}".rjust(16).ljust(40) + else: + symoff = 0 + sym_start = sample.sym_start + if sym_start is not None: + symoff = ip - sym_start + offs = f"+{symoff:#x}" if symoff != 0 else "" + start_str = common_start_str(comm, sample) + (symbol + offs).ljust(40) + + global GLB_SOURCE_FILE_NAME, GLB_LINE_NUMBER, GLB_DSO + + source_file_name, line_number, source_line, _ = sample.srccode() + if source_file_name: + if GLB_LINE_NUMBER == line_number and GLB_SOURCE_FILE_NAME == source_file_name: + src_str = "" + else: + if len(source_file_name) > 40: + src_file = f"...{source_file_name[-37:]} " + else: + src_file = source_file_name.ljust(41) + + if source_line is None: + src_str = f"{src_file}{line_number:>4d} <source not found>" + else: + src_str = f"{src_file}{line_number:>4d} {source_line}" + GLB_DSO = None + elif dso == GLB_DSO: + src_str = "" + else: + src_str = dso + GLB_DSO = dso + + GLB_LINE_NUMBER = line_number + GLB_SOURCE_FILE_NAME = source_file_name + + print(start_str, src_str) + +class TraceDisasm: + """Class to handle trace disassembly.""" + def __init__(self, cli_options: argparse.Namespace): + self.options = cli_options + self.sample_idx = -1 + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single perf event.""" + self.sample_idx += 1 + global SAMPLE_IDX + SAMPLE_IDX = self.sample_idx + + if self.options.start_time and sample.sample_time < self.options.start_time: + return + if self.options.stop_time and sample.sample_time > self.options.stop_time: + return + if self.options.start_sample and self.sample_idx < self.options.start_sample: + return + if self.options.stop_sample and self.sample_idx > self.options.stop_sample: + return + + ev_name = str(sample.evsel) + if self.options.verbose: + print(f"Event type: {ev_name}") + print_sample(sample) + + dso = sample.dso or '[unknown]' + symbol = sample.symbol or '[unknown]' + dso_bid = sample.dso_bid or '[unknown]' + dso_start = sample.map_start + dso_end = sample.map_end + map_pgoff = sample.map_pgoff or 0 + + try: + thread_info = self.session.process(sample.sample_tid) + comm = thread_info.comm() + except Exception: + comm = "[unknown]" + + cpu = sample.sample_cpu + addr = sample.sample_addr + + if CPU_DATA.get(str(cpu) + 'addr') is None: + CPU_DATA[str(cpu) + 'addr'] = addr + return + + if dso == '[unknown]': + return + + if dso_start is None or dso_end is None: + print(f"Failed to find valid dso map for dso {dso}") + return + + if ev_name.startswith("instructions"): + print_srccode(comm, sample, symbol, dso) + return + + if not ev_name.startswith("branches"): + return + + self._process_branch(sample, comm, symbol, dso, dso_bid, dso_start, dso_end, map_pgoff) + + def _process_branch(self, sample: perf.sample_event, comm: str, symbol: str, dso: str, + dso_bid: str, dso_start: int, dso_end: int, map_pgoff: int) -> None: + """Helper to process branch events.""" + cpu = sample.sample_cpu + ip = sample.sample_ip + addr = sample.sample_addr + + start_addr = CPU_DATA[str(cpu) + 'addr'] + stop_addr = ip + 4 + + # Record for previous sample packet + CPU_DATA[str(cpu) + 'addr'] = addr + + # Filter out zero start_address. Optionally identify CS_ETM_TRACE_ON packet + if start_addr == 0: + if stop_addr == 4 and self.options.verbose: + print(f"CPU{cpu}: CS_ETM_TRACE_ON packet is inserted") + return + + if start_addr < dso_start or start_addr > dso_end: + print(f"Start address {start_addr:#x} is out of range [ {dso_start:#x} .. " + f"{dso_end:#x} ] for dso {dso}") + return + + if stop_addr < dso_start or stop_addr > dso_end: + print(f"Stop address {stop_addr:#x} is out of range [ {dso_start:#x} .. " + f"{dso_end:#x} ] for dso {dso}") + return + + if self.options.objdump is not None: + if dso == "[kernel.kallsyms]" or dso_start == 0x400000: + dso_vm_start = 0 + map_pgoff_local = 0 + else: + dso_vm_start = dso_start + map_pgoff_local = map_pgoff + + dso_fname = get_dso_file_path(dso, dso_bid, self.options.vmlinux) + if path.exists(dso_fname): + print_disam(dso_fname, dso_vm_start, start_addr + map_pgoff_local, + stop_addr + map_pgoff_local, self.options.objdump) + else: + print(f"Failed to find dso {dso} for address range [ " + f"{start_addr + map_pgoff_local:#x} .. {stop_addr + map_pgoff_local:#x} ]") + + print_srccode(comm, sample, symbol, dso) + + def run(self) -> None: + """Run the trace disassembly session.""" + input_file = self.options.input or "perf.data" + if not os.path.exists(input_file): + print(f"Error: {input_file} not found.", file=sys.stderr) + sys.exit(1) + + print('ARM CoreSight Trace Data Assembler Dump') + try: + self.session = perf.session(perf.data(input_file), sample=self.process_event) + except Exception as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + self.session.process_events() + print('End') + +if __name__ == "__main__": + def int_arg(v: str) -> int: + """Helper for integer command line arguments.""" + val = int(v) + if val < 0: + raise argparse.ArgumentTypeError("Argument must be a positive integer") + return val + + arg_parser = argparse.ArgumentParser(description="ARM CoreSight Trace Dump With Disassembler") + arg_parser.add_argument("-i", "--input", help="input perf.data file") + arg_parser.add_argument("-k", "--vmlinux", + help="Set path to vmlinux file. Omit to autodetect") + arg_parser.add_argument("-d", "--objdump", nargs="?", const=default_objdump(), + help="Show disassembly. Can also be used to change the objdump path") + arg_parser.add_argument("-v", "--verbose", action="store_true", help="Enable debugging log") + arg_parser.add_argument("--start-time", type=int_arg, + help="Monotonic clock time of sample to start from.") + arg_parser.add_argument("--stop-time", type=int_arg, + help="Monotonic clock time of sample to stop at.") + arg_parser.add_argument("--start-sample", type=int_arg, + help="Index of sample to start from.") + arg_parser.add_argument("--stop-sample", type=int_arg, + help="Index of sample to stop at.") + + parsed_options = arg_parser.parse_args() + if (parsed_options.start_time and parsed_options.stop_time and \ + parsed_options.start_time >= parsed_options.stop_time): + print("--start-time must less than --stop-time") + sys.exit(2) + if (parsed_options.start_sample and parsed_options.stop_sample and \ + parsed_options.start_sample >= parsed_options.stop_sample): + print("--start-sample must less than --stop-sample") + sys.exit(2) + + td = TraceDisasm(parsed_options) + td.run()
tools/perf/tests/shell/test_arm_coresight_disasm.sh +6 -4
@@ -45,8 +45,9 @@ branch_search="\sbl${sep}b${sep}b.ne${sep}b.eq${sep}cbz\s" if [ -e /proc/kcore ]; then echo "Testing kernel disassembly" perf record -o ${perfdata} -e cs_etm//k --kcore -- touch $file > /dev/null 2>&1 - perf script -i ${perfdata} -s python:${script_path} -- \ - -d --stop-sample=30 2> /dev/null > ${file} + # shellcheck source=lib/setup_python.sh + . "$(dirname "$0")"/lib/setup_python.sh + $PYTHON ${script_path} -i ${perfdata} -d --stop-sample=30 2> /dev/null > ${file} grep -q -e ${branch_search} ${file} echo "Found kernel branches" else
@@ -57,8 +58,9 @@ fi ## Test user ## echo "Testing userspace disassembly" perf record -o ${perfdata} -e cs_etm//u -- touch $file > /dev/null 2>&1 -perf script -i ${perfdata} -s python:${script_path} -- \ - -d --stop-sample=30 2> /dev/null > ${file} +# shellcheck source=lib/setup_python.sh +. "$(dirname "$0")"/lib/setup_python.sh +$PYTHON ${script_path} -i ${perfdata} -d --stop-sample=30 2> /dev/null > ${file} grep -q -e ${branch_search} ${file} echo "Found userspace branches"
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 32/58] perf arm-cs-trace-disasm: Port arm-cs-trace-disasm to use python module
Sashiko AI review found 4 potential issue(s): - [High] Missing `import perf` causes immediate runtime crash. - [High] Runtime crash due to unpacking mismatch on `sample.srccode()`. - [Medium] Broken…
Sashiko AI review found 4 potential issue(s):
- [High] Missing import perf causes immediate runtime crash.
- [High] Runtime crash due to unpacking mismatch on sample.srccode().
- [Medium] Broken test integration due to stale script path.
- [Medium] Missing early termination causes severe performance regression. --
commit 5bdc3caf81b1523e22044cb273de9d509ecc0c13
Author: Ian Rogers <[email protected]>
perf arm-cs-trace-disasm: Port arm-cs-trace-disasm to use python module
This commit adds a port of the arm-cs-trace-disasm script that uses the perf python module directly to improve performance. It also updates the testing to use the newly ported script.
Quoted 11 lines
tools/perf/python/arm-cs-trace-disasm.py new file +5
-0@@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +arm-cs-trace-disasm.py: ARM CoreSight Trace Dump With Disassember using perf python module +"""
tools/perf/python/arm-cs-trace-disasm.py line 15
Quoted 10 lines
tools/perf/python/arm-cs-trace-disasm.py new file +10
-0+ +import os +from os import path +import re +from subprocess import check_output +import argparse +import platform +import sys +from typing import Dict, List, Optional +
Does this script need an import for the perf module? The code uses it directly in several places like perf.config_get() and perf.sample_event, but it does not appear to be imported.
[ ... ]
Quoted 2 lines
tools/perf/python/arm-cs-trace-disasm.py new file +2
-0+def print_srccode(comm: str, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print source code and symbols for a sample."""
[ ... ]
tools/perf/python/arm-cs-trace-disasm.py line 145
Quoted 3 lines
tools/perf/python/arm-cs-trace-disasm.py new file +3
-0+ global GLB_SOURCE_FILE_NAME, GLB_LINE_NUMBER, GLB_DSO + + source_file_name, line_number, source_line, _ = sample.srccode()
Can this cause a runtime error when unpacking? The underlying pyrf_sample_event__srccode in python.c constructs a 3-element tuple using Py_BuildValue("(sIs#)", ...), but this code attempts to unpack 4 elements.
[ ... ]
Quoted 11 lines
tools/perf/python/arm-cs-trace-disasm.py new file +11
-0+class TraceDisasm: + """Class to handle trace disassembly.""" + def __init__(self, cli_options: argparse.Namespace): + self.options = cli_options + self.sample_idx = -1 + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single perf event.""" + self.sample_idx += 1 + global SAMPLE_IDX
tools/perf/python/arm-cs-trace-disasm.py line 191
Quoted 10 lines
tools/perf/python/arm-cs-trace-disasm.py new file +10
-0+ SAMPLE_IDX = self.sample_idx + + if self.options.start_time and sample.sample_time < self.options.start_time: + return + if self.options.stop_time and sample.sample_time > self.options.stop_time: + return + if self.options.start_sample and self.sample_idx < self.options.start_sample: + return + if self.options.stop_sample and self.sample_idx > self.options.stop_sample: + return
Does returning here correctly stop the analysis? Since self.session.process_events() loops through all records, returning only skips the current callback instead of terminating the session like the original script did.
[ ... ]
Quoted 4 lines
tools/perf/tests/shell/test_arm_coresight_disasm.sh +0
-0
[ ... ]
tools/perf/tests/shell/test_arm_coresight_disasm.sh branch_search="\sbl${sep}b${sep}b.ne${sep}b.eq${sep}cbz\s" · line 50
Quoted 9 lines
tools/perf/tests/shell/test_arm_coresight_disasm.sh +3
-2@@ -45,8 +45,9 @@ branch_search="\sbl${sep}b${sep}b.ne${sep}b.eq${sep}cbz\s" if [ -e /proc/kcore ]; then echo "Testing kernel disassembly" perf record -o ${perfdata} -e cs_etm//k --kcore -- touch $file > /dev/null 2>&1 - perf script -i ${perfdata} -s python:${script_path} -- \ - -d --stop-sample=30 2> /dev/null > ${file} + # shellcheck source=lib/setup_python.sh + . "$(dirname "$0")"/lib/setup_python.sh + $PYTHON ${script_path} -i ${perfdata} -d --stop-sample=30 2> /dev/null > ${file}
Will this test fail because script_path still points to the old legacy script? The new python module was added to tools/perf/python/ instead of tools/perf/scripts/python/.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=32
Ian Rogers
reviewerPATCH 33/58 perf check-perf-trace: Port check-perf-trace to use python module
In reply to Ian RogersAdd a port of the check-perf-trace script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
--- tools/perf/python/check-perf-trace.py | 111 ++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 tools/perf/python/check-perf-trace.py
tools/perf/python/check-perf-trace.py new file +111 -0
@@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Basic test of Python scripting support for perf. +Ported from tools/perf/scripts/python/check-perf-trace.py +""" + +import argparse +import collections +import perf + +unhandled: collections.defaultdict[str, int] = collections.defaultdict(int) + +softirq_vecs = { + 0: "HI_SOFTIRQ", + 1: "TIMER_SOFTIRQ", + 2: "NET_TX_SOFTIRQ", + 3: "NET_RX_SOFTIRQ", + 4: "BLOCK_SOFTIRQ", + 5: "IRQ_POLL_SOFTIRQ", + 6: "TASKLET_SOFTIRQ", + 7: "SCHED_SOFTIRQ", + 8: "HRTIMER_SOFTIRQ", + 9: "RCU_SOFTIRQ", +} + +def trace_begin() -> None: + """Called at the start of trace processing.""" + print("trace_begin") + +def trace_end() -> None: + """Called at the end of trace processing.""" + print_unhandled() + +def symbol_str(event_name: str, field_name: str, value: int) -> str: + """Resolves symbol values to strings.""" + if event_name == "irq__softirq_entry" and field_name == "vec": + return softirq_vecs.get(value, str(value)) + return str(value) + +def flag_str(event_name: str, field_name: str, value: int) -> str: + """Resolves flag values to strings.""" + if event_name == "kmem__kmalloc" and field_name == "gfp_flags": + return f"0x{value:x}" + return str(value) + +def print_header(event_name: str, sample: perf.sample_event) -> None: + """Prints common header for events.""" + secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000 + print(f"{event_name:<20} {sample.sample_cpu:5} {secs:05}.{nsecs:09} " + f"{sample.sample_pid:8} {sample.sample_comm:<20} ", end=' ') + +def print_uncommon(sample: perf.sample_event) -> None: + """Prints uncommon fields for tracepoints.""" + # Fallback to 0 if field not found (e.g. on older kernels or if not tracepoint) + pc = getattr(sample, "common_preempt_count", 0) + flags = getattr(sample, "common_flags", 0) + lock_depth = getattr(sample, "common_lock_depth", 0) + + print(f"common_preempt_count={pc}, common_flags={flags}, " + f"common_lock_depth={lock_depth}, ") + +def irq__softirq_entry(sample: perf.sample_event) -> None: + """Handles irq:softirq_entry events.""" + print_header("irq__softirq_entry", sample) + print_uncommon(sample) + print(f"vec={symbol_str('irq__softirq_entry', 'vec', sample.vec)}") + +def kmem__kmalloc(sample: perf.sample_event) -> None: + """Handles kmem:kmalloc events.""" + print_header("kmem__kmalloc", sample) + print_uncommon(sample) + + print(f"call_site={sample.call_site:d}, ptr={sample.ptr:d}, " + f"bytes_req={sample.bytes_req:d}, bytes_alloc={sample.bytes_alloc:d}, " + f"gfp_flags={flag_str('kmem__kmalloc', 'gfp_flags', sample.gfp_flags)}") + +def trace_unhandled(event_name: str) -> None: + """Tracks unhandled events.""" + unhandled[event_name] += 1 + +def print_unhandled() -> None: + """Prints summary of unhandled events.""" + if not unhandled: + return + print("\nunhandled events:\n") + print(f"{'event':<40} {'count':>10}") + print("---------------------------------------- -----------") + for event_name, count in unhandled.items(): + print(f"{event_name:<40} {count:10}") + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + event_name = str(sample.evsel) + if "evsel(irq:softirq_entry)" in event_name: + irq__softirq_entry(sample) + elif "evsel(kmem:kmalloc)" in event_name: + kmem__kmalloc(sample) + else: + trace_unhandled(event_name) + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + trace_begin() + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + trace_end()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 33/58] perf check-perf-trace: Port check-perf-trace to use python module
Sashiko AI review found 1 potential issue(s): - [Medium] The script accesses a non-existent `sample_comm` attribute on `perf.sample_event` objects, causing an unconditional `AttributeError` at…
Sashiko AI review found 1 potential issue(s):
- [Medium] The script accesses a non-existent sample_comm attribute on perf.sample_event objects, causing an unconditional AttributeError at runtime. --
commit c7ead2d8a935aefa471da2549e7433796e508f75
Author: Ian Rogers <[email protected]>
perf check-perf-trace: Port check-perf-trace to use python module
This commit adds a python script port of check-perf-trace that uses the perf python module directly. This approach improves performance by avoiding the creation of intermediate dictionaries for event fields.
Quoted 5 lines
tools/perf/python/check-perf-trace.py new file +0
-0
[ ... ]
tools/perf/python/check-perf-trace.py line 52
Quoted 6 lines
tools/perf/python/check-perf-trace.py new file +6
-0+def print_header(event_name: str, sample: perf.sample_event) -> None: + """Prints common header for events.""" + secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000 + print(f"{event_name:<20} {sample.sample_cpu:5} {secs:05}.{nsecs:09} " + f"{sample.sample_pid:8} {sample.sample_comm:<20} ", end=' ')
Will this raise an AttributeError when accessing sample.sample_comm?
The perf.sample_event C-extension wrapper implemented in tools/perf/util/python.c does not appear to expose a sample_comm attribute or property.
Because sample_comm is also not a tracepoint field, the dynamic fallback in get_tracepoint_field() will likely fail, causing PyObject_GenericGetAttr() to raise an exception and stop the script when processing the first event.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=33
Ian Rogers
reviewerPATCH 34/58 perf compaction-times: Port compaction-times to use python module
In reply to Ian RogersAdd a port of the compaction-times script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
--- tools/perf/python/compaction-times.py | 325 ++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100755 tools/perf/python/compaction-times.py
tools/perf/python/compaction-times.py new file +325 -0
@@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Report time spent in memory compaction. + +Memory compaction is a feature in the Linux kernel that defragments memory +by moving used pages to create larger contiguous blocks of free memory. This +is particularly useful for allocating huge pages. + +This script processes trace events related to memory compaction and reports: +- Total time spent in compaction (stall time). +- Statistics for page migration (moved vs. failed). +- Statistics for the free scanner (scanned vs. isolated pages). +- Statistics for the migration scanner (scanned vs. isolated pages). + +Definitions: +- **Compaction**: Defragmenting memory by moving allocated pages. +- **Migration**: Moving pages from their current location to free pages found by the free scanner. +- **Free Scanner**: Scans memory (typically from the end of a zone) to find free pages. +- **Migration Scanner**: Scans memory (typically from the beginning of a zone) + to find pages to move. +- **Isolated Pages**: Pages that have been temporarily removed from the buddy + system for migration or as migration targets. + +Ported from tools/perf/scripts/python/compaction-times.py to the modern perf Python module. +""" + +import argparse +import enum +import re +import sys +from typing import Callable, Dict, List, Optional, Any +import perf + +class Popt(enum.IntEnum): + """Process display options.""" + DISP_DFL = 0 + DISP_PROC = 1 + DISP_PROC_VERBOSE = 2 + +class Topt(enum.IntFlag): + """Trace display options.""" + DISP_TIME = 0 + DISP_MIG = 1 + DISP_ISOLFREE = 2 + DISP_ISOLMIG = 4 + DISP_ALL = DISP_MIG | DISP_ISOLFREE | DISP_ISOLMIG + +# Globals to satisfy pylint when accessed in functions before assignment in main. +OPT_NS = True +opt_disp = Topt.DISP_ALL +opt_proc = Popt.DISP_DFL + +def get_comm_filter(regex: re.Pattern) -> Callable[[int, str], bool]: + """Returns a filter function based on command regex.""" + def filter_func(_pid: int, comm: str) -> bool: + regex_match = regex.search(comm) + return regex_match is None or regex_match.group() == "" + return filter_func + +def get_pid_filter(low_str: str, high_str: str) -> Callable[[int, str], bool]: + """Returns a filter function based on PID range.""" + low = 0 if low_str == "" else int(low_str) + high = 0 if high_str == "" else int(high_str) + + def filter_func(pid: int, _comm: str) -> bool: + return not (pid >= low and (high == 0 or pid <= high)) + return filter_func + +def ns_to_time(ns: int) -> str: + """Format nanoseconds to string based on options.""" + return f"{ns}ns" if OPT_NS else f"{round(ns, -3) // 1000}us" + +class Pair: + """Represents a pair of related counters (e.g., scanned vs isolated, moved vs failed).""" + def __init__(self, aval: int, bval: int, + alabel: Optional[str] = None, blabel: Optional[str] = None): + self.alabel = alabel + self.blabel = blabel + self.aval = aval + self.bval = bval + + def __add__(self, rhs: 'Pair') -> 'Pair': + self.aval += rhs.aval + self.bval += rhs.bval + return self + + def __str__(self) -> str: + return f"{self.alabel}={self.aval} {self.blabel}={self.bval}" + +class Cnode: + """Holds statistics for a single compaction event or an aggregated set of events.""" + def __init__(self, ns: int): + self.ns = ns + self.migrated = Pair(0, 0, "moved", "failed") + self.fscan = Pair(0, 0, "scanned", "isolated") + self.mscan = Pair(0, 0, "scanned", "isolated") + + def __add__(self, rhs: 'Cnode') -> 'Cnode': + self.ns += rhs.ns + self.migrated += rhs.migrated + self.fscan += rhs.fscan + self.mscan += rhs.mscan + return self + + def __str__(self) -> str: + prev = False + s = f"{ns_to_time(self.ns)} " + if opt_disp & Topt.DISP_MIG: + s += f"migration: {self.migrated}" + prev = True + if opt_disp & Topt.DISP_ISOLFREE: + s += f"{' ' if prev else ''}free_scanner: {self.fscan}" + prev = True + if opt_disp & Topt.DISP_ISOLMIG: + s += f"{' ' if prev else ''}migration_scanner: {self.mscan}" + return s + + def complete(self, secs: int, nsecs: int) -> None: + """Complete the node with duration.""" + self.ns = (secs * 1000000000 + nsecs) - self.ns + + def increment(self, migrated: Optional[Pair], fscan: Optional[Pair], + mscan: Optional[Pair]) -> None: + """Increment statistics.""" + if migrated is not None: + self.migrated += migrated + if fscan is not None: + self.fscan += fscan + if mscan is not None: + self.mscan += mscan + +class Chead: + """Aggregates compaction statistics per process (PID) and maintains total statistics.""" + heads: Dict[int, 'Chead'] = {} + val = Cnode(0) + fobj: Optional[Any] = None + + @classmethod + def add_filter(cls, fobj: Any) -> None: + """Add a filter object.""" + cls.fobj = fobj + + @classmethod + def create_pending(cls, pid: int, comm: str, start_secs: int, start_nsecs: int) -> None: + """Create a pending node for a process.""" + filtered = False + try: + head = cls.heads[pid] + filtered = head.is_filtered() + except KeyError: + if cls.fobj is not None: + filtered = cls.fobj.filter(pid, comm) + head = cls.heads[pid] = Chead(comm, pid, filtered) + + if not filtered: + head.mark_pending(start_secs, start_nsecs) + + @classmethod + def increment_pending(cls, pid: int, migrated: Optional[Pair], + fscan: Optional[Pair], mscan: Optional[Pair]) -> None: + """Increment pending stats for a process.""" + if pid not in cls.heads: + return + head = cls.heads[pid] + if not head.is_filtered(): + if head.is_pending(): + head.do_increment(migrated, fscan, mscan) + else: + sys.stderr.write(f"missing start compaction event for pid {pid}\n") + + @classmethod + def complete_pending(cls, pid: int, secs: int, nsecs: int) -> None: + """Complete pending stats for a process.""" + if pid not in cls.heads: + return + head = cls.heads[pid] + if not head.is_filtered(): + if head.is_pending(): + head.make_complete(secs, nsecs) + else: + sys.stderr.write(f"missing start compaction event for pid {pid}\n") + + @classmethod + def gen(cls): + """Generate heads for display.""" + if opt_proc != Popt.DISP_DFL: + yield from cls.heads.values() + + @classmethod + def get_total(cls) -> Cnode: + """Get total statistics.""" + return cls.val + + def __init__(self, comm: str, pid: int, filtered: bool): + self.comm = comm + self.pid = pid + self.val = Cnode(0) + self.pending: Optional[Cnode] = None + self.filtered = filtered + self.list: List[Cnode] = [] + + def mark_pending(self, secs: int, nsecs: int) -> None: + """Mark node as pending.""" + self.pending = Cnode(secs * 1000000000 + nsecs) + + def do_increment(self, migrated: Optional[Pair], fscan: Optional[Pair], + mscan: Optional[Pair]) -> None: + """Increment pending stats.""" + if self.pending is not None: + self.pending.increment(migrated, fscan, mscan) + + def make_complete(self, secs: int, nsecs: int) -> None: + """Make pending stats complete.""" + if self.pending is not None: + self.pending.complete(secs, nsecs) + Chead.val += self.pending + + if opt_proc != Popt.DISP_DFL: + self.val += self.pending + + if opt_proc == Popt.DISP_PROC_VERBOSE: + self.list.append(self.pending) + self.pending = None + + def enumerate(self) -> None: + """Enumerate verbose stats.""" + if opt_proc == Popt.DISP_PROC_VERBOSE and not self.is_filtered(): + for i, pelem in enumerate(self.list): + sys.stdout.write(f"{self.pid}[{self.comm}].{i+1}: {pelem}\n") + + def is_pending(self) -> bool: + """Check if node is pending.""" + return self.pending is not None + + def is_filtered(self) -> bool: + """Check if node is filtered.""" + return self.filtered + + def display(self) -> None: + """Display stats.""" + if not self.is_filtered(): + sys.stdout.write(f"{self.pid}[{self.comm}]: {self.val}\n") + +def trace_end() -> None: + """Called at the end of trace processing.""" + sys.stdout.write(f"total: {Chead.get_total()}\n") + for i in Chead.gen(): + i.display() + i.enumerate() + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + event_name = str(sample.evsel) + pid = sample.sample_pid + comm = sample.sample_comm + secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000 + + if "evsel(compaction:mm_compaction_begin)" in event_name: + Chead.create_pending(pid, comm, secs, nsecs) + elif "evsel(compaction:mm_compaction_end)" in event_name: + Chead.complete_pending(pid, secs, nsecs) + elif "evsel(compaction:mm_compaction_migratepages)" in event_name: + Chead.increment_pending(pid, Pair(sample.nr_migrated, sample.nr_failed), None, None) + elif "evsel(compaction:mm_compaction_isolate_freepages)" in event_name: + Chead.increment_pending(pid, None, Pair(sample.nr_scanned, sample.nr_taken), None) + elif "evsel(compaction:mm_compaction_isolate_migratepages)" in event_name: + Chead.increment_pending(pid, None, None, Pair(sample.nr_scanned, sample.nr_taken)) + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Report time spent in compaction") + ap.add_argument("-p", action="store_true", help="display by process") + ap.add_argument("-pv", action="store_true", help="display by process (verbose)") + ap.add_argument("-u", action="store_true", help="display results in microseconds") + ap.add_argument("-t", action="store_true", help="display stall times only") + ap.add_argument("-m", action="store_true", help="display stats for migration") + ap.add_argument("-fs", action="store_true", help="display stats for free scanner") + ap.add_argument("-ms", action="store_true", help="display stats for migration scanner") + ap.add_argument("filter", nargs="?", help="pid|pid-range|comm-regex") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + opt_proc = Popt.DISP_DFL + if args.pv: + opt_proc = Popt.DISP_PROC_VERBOSE + elif args.p: + opt_proc = Popt.DISP_PROC + + OPT_NS = not args.u + + opt_disp = Topt.DISP_ALL + if args.t or args.m or args.fs or args.ms: + opt_disp = Topt(0) + if args.t: + opt_disp |= Topt.DISP_TIME + if args.m: + opt_disp |= Topt.DISP_MIG + if args.fs: + opt_disp |= Topt.DISP_ISOLFREE + if args.ms: + opt_disp |= Topt.DISP_ISOLMIG + + if args.filter: + PID_PATTERN = r"^(\d*)-(\d*)$|^(\d*)$" + pid_re = re.compile(PID_PATTERN) + match = pid_re.search(args.filter) + filter_obj: Any = None + if match is not None and match.group() != "": + if match.group(3) is not None: + filter_obj = get_pid_filter(match.group(3), match.group(3)) + else: + filter_obj = get_pid_filter(match.group(1), match.group(2)) + else: + try: + comm_re = re.compile(args.filter) + except re.error: + sys.stderr.write(f"invalid regex '{args.filter}'\n") + sys.exit(1) + filter_obj = get_comm_filter(comm_re) + Chead.add_filter(filter_obj) + + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + trace_end()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 34/58] perf compaction-times: Port compaction-times to use python module
Sashiko AI review found 2 potential issue(s): - [High] Accessing `sample.sample_comm` crashes the script with an AttributeError because the property is not implemented in the `perf` Python C…
Sashiko AI review found 2 potential issue(s):
- [High] Accessing sample.sample_comm crashes the script with an AttributeError because the property is not implemented in the perf Python C extension.
- [High] Calling .filter() on a Python function object causes an AttributeError, crashing the script when a user supplies a filter argument. --
commit e19ed9f5bdaeeb7d6732e01b049d84fb4e856fc0
Author: Ian Rogers <[email protected]>
perf compaction-times: Port compaction-times to use python module
This commit adds a port of the compaction-times script that uses the perf python module directly to improve performance by avoiding intermediate dictionaries.
Quoted 5 lines
tools/perf/python/compaction-times.py new file +0
-0
[ ... ]
tools/perf/python/compaction-times.py line 153
Quoted 10 lines
tools/perf/python/compaction-times.py new file +10
-0+ @classmethod + def create_pending(cls, pid: int, comm: str, start_secs: int, start_nsecs: int) -> None: + """Create a pending node for a process.""" + filtered = False + try: + head = cls.heads[pid] + filtered = head.is_filtered() + except KeyError: + if cls.fobj is not None: + filtered = cls.fobj.filter(pid, comm)
Does calling .filter() on cls.fobj cause an AttributeError?
The helper functions get_comm_filter() and get_pid_filter() return standard Python closures (functions) rather than class instances.
Since a function object does not have a .filter() method, it appears this will crash the script when a user supplies a filter argument.
Should this be invoked directly as cls.fobj(pid, comm)?
Quoted 4 lines
tools/perf/python/compaction-times.py new file +4
-0+ head = cls.heads[pid] = Chead(comm, pid, filtered) + + if not filtered: + head.mark_pending(start_secs, start_nsecs)
[ ... ]
tools/perf/python/compaction-times.py line 256
Quoted 5 lines
tools/perf/python/compaction-times.py new file +5
-0+def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + event_name = str(sample.evsel) + pid = sample.sample_pid + comm = sample.sample_comm
Will accessing sample.sample_comm crash the script with an AttributeError?
The perf Python C extension does not implement a sample_comm member or getter in pyrf_sample_event__members or pyrf_sample_event__getset.
The underlying tracepoint exposes the command line string as common_comm, which can be accessed via dynamic tracepoint field resolution.
Because this property is missing, accessing it will raise an AttributeError on the first processed event.
Quoted 2 lines
tools/perf/python/compaction-times.py new file +2
-0+ secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=34
Ian Rogers
reviewerPATCH 35/58 perf event_analyzing_sample: Port event_analyzing_sample to use python module
In reply to Ian RogersAdd a port of the event_analyzing_sample script that uses the perf python module directly. This approach is significantly faster than using perf script callbacks as it avoids creating intermediate dictionaries for all event fields.
--- tools/perf/python/event_analyzing_sample.py | 291 ++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100755 tools/perf/python/event_analyzing_sample.py
tools/perf/python/event_analyzing_sample.py new file +291 -0
@@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +General event handler in Python, using SQLite to analyze events. + +The 2 database related functions in this script just show how to gather +the basic information, and users can modify and write their own functions +according to their specific requirement. + +The first function "show_general_events" just does a basic grouping for all +generic events with the help of sqlite, and the 2nd one "show_pebs_ll" is +for a x86 HW PMU event: PEBS with load latency data. + +Ported from tools/perf/scripts/python/event_analyzing_sample.py +""" + +import argparse +import math +import os +import sqlite3 +import struct +from typing import Any +import perf + +# Event types, user could add more here +EVTYPE_GENERIC = 0 +EVTYPE_PEBS = 1 # Basic PEBS event +EVTYPE_PEBS_LL = 2 # PEBS event with load latency info +EVTYPE_IBS = 3 + +# +# Currently we don't have good way to tell the event type, but by +# the size of raw buffer, raw PEBS event with load latency data's +# size is 176 bytes, while the pure PEBS event's size is 144 bytes. +# +def create_event(name, comm, dso, symbol, raw_buf): + """Create an event object based on raw buffer size.""" + if len(raw_buf) == 144: + event = PebsEvent(name, comm, dso, symbol, raw_buf) + elif len(raw_buf) == 176: + event = PebsNHM(name, comm, dso, symbol, raw_buf) + else: + event = PerfEvent(name, comm, dso, symbol, raw_buf) + + return event + +class PerfEvent: + """Base class for all perf event samples.""" + event_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_GENERIC): + self.name = name + self.comm = comm + self.dso = dso + self.symbol = symbol + self.raw_buf = raw_buf + self.ev_type = ev_type + PerfEvent.event_num += 1 + + def show(self): + """Display PMU event info.""" + print(f"PMU event: name={self.name:12s}, symbol={self.symbol:24s}, " + f"comm={self.comm:8s}, dso={self.dso:12s}") + +# +# Basic Intel PEBS (Precise Event-based Sampling) event, whose raw buffer +# contains the context info when that event happened: the EFLAGS and +# linear IP info, as well as all the registers. +# +class PebsEvent(PerfEvent): + """Intel PEBS event.""" + pebs_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_PEBS): + tmp_buf = raw_buf[0:80] + flags, ip, ax, bx, cx, dx, si, di, bp, sp = struct.unpack('QQQQQQQQQQ', tmp_buf) + self.flags = flags + self.ip = ip + self.ax = ax + self.bx = bx + self.cx = cx + self.dx = dx + self.si = si + self.di = di + self.bp = bp + self.sp = sp + + super().__init__(name, comm, dso, symbol, raw_buf, ev_type) + PebsEvent.pebs_num += 1 + del tmp_buf + +# +# Intel Nehalem and Westmere support PEBS plus Load Latency info which lie +# in the four 64 bit words write after the PEBS data: +# Status: records the IA32_PERF_GLOBAL_STATUS register value +# DLA: Data Linear Address (EIP) +# DSE: Data Source Encoding, where the latency happens, hit or miss +# in L1/L2/L3 or IO operations +# LAT: the actual latency in cycles +# +class PebsNHM(PebsEvent): + """Intel Nehalem/Westmere PEBS event with load latency.""" + pebs_nhm_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_PEBS_LL): + tmp_buf = raw_buf[144:176] + status, dla, dse, lat = struct.unpack('QQQQ', tmp_buf) + self.status = status + self.dla = dla + self.dse = dse + self.lat = lat + + super().__init__(name, comm, dso, symbol, raw_buf, ev_type) + PebsNHM.pebs_nhm_num += 1 + del tmp_buf + +session: Any = None + +# Database connection +# If the perf.data has a big number of samples, then the insert operation +# will be very time consuming (about 10+ minutes for 10000 samples) if the +# .db database is on disk. Move the .db file to RAM based FS to speedup +# the handling, which will cut the time down to several seconds. +DB_PATH = "/dev/shm/perf.db" if os.path.exists("/dev/shm") else "perf.db" +con = sqlite3.connect(DB_PATH) +con.isolation_level = None + +def trace_begin() -> None: + """Initialize database tables.""" + print("In trace_begin:\n") + + # Will create several tables at the start, pebs_ll is for PEBS data with + # load latency info, while gen_events is for general event. + con.execute(""" + create table if not exists gen_events ( + name text, + symbol text, + comm text, + dso text + );""") + con.execute(""" + create table if not exists pebs_ll ( + name text, + symbol text, + comm text, + dso text, + flags integer, + ip integer, + status integer, + dse integer, + dla integer, + lat integer + );""") + +def insert_db(event: Any) -> None: + """Insert event into database.""" + if event.ev_type == EVTYPE_GENERIC: + con.execute("insert into gen_events values(?, ?, ?, ?)", + (event.name, event.symbol, event.comm, event.dso)) + elif event.ev_type == EVTYPE_PEBS_LL: + event.ip &= 0x7fffffffffffffff + event.dla &= 0x7fffffffffffffff + con.execute("insert into pebs_ll values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (event.name, event.symbol, event.comm, event.dso, event.flags, + event.ip, event.status, event.dse, event.dla, event.lat)) + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + # Create and insert event object to a database so that user could + # do more analysis with simple database commands. + + # Resolve comm, symbol, dso + comm = "Unknown_comm" + try: + if session is not None: + proc = session.process(sample.sample_pid) + if proc: + comm = proc.comm() + except TypeError: + pass + + # Symbol and dso info are not always resolved + dso = sample.dso if hasattr(sample, 'dso') and sample.dso else "Unknown_dso" + symbol = sample.symbol if hasattr(sample, 'symbol') and sample.symbol else "Unknown_symbol" + name = str(sample.evsel) + + # Create the event object and insert it to the right table in database + try: + event = create_event(name, comm, dso, symbol, sample.raw_buf) + insert_db(event) + except (sqlite3.Error, ValueError, TypeError) as e: + print(f"Error creating/inserting event: {e}") + +def num2sym(num: int) -> str: + """Convert number to a histogram symbol (log2).""" + # As the event number may be very big, so we can't use linear way + # to show the histogram in real number, but use a log2 algorithm. + if num <= 0: + return "" + snum = '#' * (int(math.log(num, 2)) + 1) + return snum + +def show_general_events() -> None: + """Display statistics for general events.""" + count = con.execute("select count(*) from gen_events") + for t in count: + print(f"There is {t[0]} records in gen_events table") + if t[0] == 0: + return + + print("Statistics about the general events grouped by thread/symbol/dso: \n") + + # Group by thread + commq = con.execute(""" + select comm, count(comm) from gen_events + group by comm order by -count(comm) + """) + print(f"\n{ 'comm':>16} {'number':>8} {'histogram':>16}\n{'='*42}") + for row in commq: + print(f"{row[0]:>16} {row[1]:>8} {num2sym(row[1])}") + + # Group by symbol + print(f"\n{'symbol':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + symbolq = con.execute(""" + select symbol, count(symbol) from gen_events + group by symbol order by -count(symbol) + """) + for row in symbolq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by dso + print(f"\n{'dso':>40} {'number':>8} {'histogram':>16}\n{'='*74}") + dsoq = con.execute("select dso, count(dso) from gen_events group by dso order by -count(dso)") + for row in dsoq: + print(f"{row[0]:>40} {row[1]:>8} {num2sym(row[1])}") + +def show_pebs_ll() -> None: + """Display statistics for PEBS load latency events.""" + # This function just shows the basic info, and we could do more with the + # data in the tables, like checking the function parameters when some + # big latency events happen. + count = con.execute("select count(*) from pebs_ll") + for t in count: + print(f"There is {t[0]} records in pebs_ll table") + if t[0] == 0: + return + + print("Statistics about the PEBS Load Latency events grouped by thread/symbol/dse/latency: \n") + + # Group by thread + commq = con.execute("select comm, count(comm) from pebs_ll group by comm order by -count(comm)") + print(f"\n{'comm':>16} {'number':>8} {'histogram':>16}\n{'='*42}") + for row in commq: + print(f"{row[0]:>16} {row[1]:>8} {num2sym(row[1])}") + + # Group by symbol + print(f"\n{'symbol':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + symbolq = con.execute(""" + select symbol, count(symbol) from pebs_ll + group by symbol order by -count(symbol) + """) + for row in symbolq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by dse + dseq = con.execute("select dse, count(dse) from pebs_ll group by dse order by -count(dse)") + print(f"\n{'dse':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + for row in dseq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by latency + latq = con.execute("select lat, count(lat) from pebs_ll group by lat order by lat") + print(f"\n{'latency':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + for row in latq: + print(f"{str(row[0]):>32} {row[1]:>8} {num2sym(row[1])}") + +def trace_end() -> None: + """Called at the end of trace processing.""" + print("In trace_end:\n") + show_general_events() + show_pebs_ll() + con.close() + if os.path.exists(DB_PATH) and DB_PATH != "perf.db": + os.remove(DB_PATH) + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Analyze events with SQLite") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + trace_begin() + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + trace_end()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 35/58] perf event_analyzing_sample: Port event_analyzing_sample to use python module
Sashiko AI review found 3 potential issue(s): - [Critical] The script uses a hardcoded, predictable file path in a shared, world-writable directory (`/dev/shm/perf.db`), exposing a symlink attack…
Sashiko AI review found 3 potential issue(s):
- [Critical] The script uses a hardcoded, predictable file path in a shared, world-writable directory (/dev/shm/perf.db), exposing a symlink attack vulnerability.
- [Medium] Event names in the database are wrapped in 'evsel(...)', altering the expected output and breaking compatibility.
- [Medium] Thread command name (comm) resolution fails or returns incorrect values for sub-threads.
--
commit 3f169dabd50cb00496ff1c84ac713efcfbc9dee4
Author: Ian Rogers <[email protected]>
perf event_analyzing_sample: Port event_analyzing_sample to use python module
This commit adds a port of the event_analyzing_sample script to use the perf python module directly. It aims to improve performance by avoiding the creation of intermediate dictionaries for all event fields.
Quoted 6 lines
tools/perf/python/event_analyzing_sample.py new file +0
-0@@ -0,0 +1,291 @@
[ ... ]
tools/perf/python/event_analyzing_sample.py line 122
Quoted 9 lines
tools/perf/python/event_analyzing_sample.py new file +9
-0+session: Any = None + +# Database connection +# If the perf.data has a big number of samples, then the insert operation +# will be very time consuming (about 10+ minutes for 10000 samples) if the +# .db database is on disk. Move the .db file to RAM based FS to speedup +# the handling, which will cut the time down to several seconds. +DB_PATH = "/dev/shm/perf.db" if os.path.exists("/dev/shm") else "perf.db" +con = sqlite3.connect(DB_PATH)
This was retained from the original script, but does this create a symlink overwrite regression?
Since /dev/shm is world-writable, an unprivileged user could pre-create a symlink at /dev/shm/perf.db pointing to an arbitrary system file. If a privileged user runs this script, would sqlite3.connect() follow the symlink and overwrite the target file?
[ ... ]
tools/perf/python/event_analyzing_sample.py line 173
Quoted 10 lines
tools/perf/python/event_analyzing_sample.py new file +10
-0+def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + # Create and insert event object to a database so that user could + # do more analysis with simple database commands. + + # Resolve comm, symbol, dso + comm = "Unknown_comm" + try: + if session is not None: + proc = session.process(sample.sample_pid)
Does this code successfully resolve the command name for sub-threads?
Looking at the C extension's implementation, pyrf_session__process passes the single sample_pid value to machine__find_thread() for both the pid and tid arguments. This means it only finds the thread object if the TID equals the PID.
Will this cause events originating from sub-threads to record the main thread's comm or fail to resolve entirely, whereas the original callback behavior resolved the specific sub-thread correctly?
tools/perf/python/event_analyzing_sample.py line 182
Quoted 9 lines
tools/perf/python/event_analyzing_sample.py new file +9
-0+ if proc: + comm = proc.comm() + except TypeError: + pass + + # Symbol and dso info are not always resolved + dso = sample.dso if hasattr(sample, 'dso') and sample.dso else "Unknown_dso" + symbol = sample.symbol if hasattr(sample, 'symbol') and sample.symbol else "Unknown_symbol" + name = str(sample.evsel)
Does calling str() on the evsel object introduce a regression in how event names are stored?
The python type implementation for evsel formats the string representation as evsel(name). Unlike the original script which used the raw event name like cycles, this code will record the string as evsel(cycles) in the database's name column.
Will this implicit behavioral change break compatibility for downstream database queries expecting the raw event name?
Quoted 4 lines
tools/perf/python/event_analyzing_sample.py new file +4
-0+ + # Create the event object and insert it to the right table in database + try: + event = create_event(name, comm, dso, symbol, sample.raw_buf)
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=35
Ian Rogers
reviewerPATCH 36/58 perf export-to-sqlite: Port export-to-sqlite to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/export-to-sqlite.py to use the perf Python module API.
Key changes:
- Switched from PySide2.QtSql to standard library sqlite3 for database operations.
- Implemented lazy population of lookup tables (threads, comms, dsos, symbols) from sample data.
- Added callchain support for building call paths.
--- tools/perf/python/export-to-sqlite.py | 351 ++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100755 tools/perf/python/export-to-sqlite.py
tools/perf/python/export-to-sqlite.py new file +351 -0
@@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Export perf data to a sqlite3 database. + +This script has been ported to use the modern perf Python module and the +standard library sqlite3 module. It no longer requires PySide2 or QtSql +for exporting. + +Examples of using this script with Intel PT: + + $ perf record -e intel_pt//u ls + $ python tools/perf/python/export-to-sqlite.py -i perf.data -o pt_example + +To browse the database, sqlite3 can be used e.g. + + $ sqlite3 pt_example + sqlite> .header on + sqlite> select * from samples_view where id < 10; + sqlite> .mode column + sqlite> select * from samples_view where id < 10; + sqlite> .tables + sqlite> .schema samples_view + sqlite> .quit + +An example of using the database is provided by the script +exported-sql-viewer.py. Refer to that script for details. + +Ported from tools/perf/scripts/python/export-to-sqlite.py +""" + +import argparse +import os +import sqlite3 +import sys +from typing import Dict, Optional +import perf + + +class DatabaseExporter: + """Handles database connection and exporting of perf events.""" + + def __init__(self, db_path: str): + self.con = sqlite3.connect(db_path) + self.session: Optional[perf.session] = None + + # Caches and counters grouped to reduce instance attributes + self.caches: Dict[str, dict] = { + 'threads': {}, + 'comms': {}, + 'dsos': {}, + 'symbols': {}, + 'events': {}, + 'branch_types': {}, + 'call_paths': {} + } + + self.next_id = { + 'thread': 1, + 'comm': 1, + 'dso': 1, + 'symbol': 1, + 'event': 1, + 'branch_type': 1, + 'call_path': 1 + } + + self.create_tables() + + def create_tables(self) -> None: + """Create database tables.""" + self.con.execute(""" + CREATE TABLE IF NOT EXISTS selected_events ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(80)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS machines ( + id INTEGER NOT NULL PRIMARY KEY, + pid INTEGER, + root_dir VARCHAR(4096)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS threads ( + id INTEGER NOT NULL PRIMARY KEY, + machine_id BIGINT, + process_id BIGINT, + pid INTEGER, + tid INTEGER) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS comms ( + id INTEGER NOT NULL PRIMARY KEY, + comm VARCHAR(16), + c_thread_id BIGINT, + c_time BIGINT, + exec_flag BOOLEAN) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS comm_threads ( + id INTEGER NOT NULL PRIMARY KEY, + comm_id BIGINT, + thread_id BIGINT) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS dsos ( + id INTEGER NOT NULL PRIMARY KEY, + machine_id BIGINT, + short_name VARCHAR(256), + long_name VARCHAR(4096), + build_id VARCHAR(64)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS symbols ( + id INTEGER NOT NULL PRIMARY KEY, + dso_id BIGINT, + sym_start BIGINT, + sym_end BIGINT, + binding INTEGER, + name VARCHAR(2048)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS branch_types ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(80)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS samples ( + id INTEGER NOT NULL PRIMARY KEY, + evsel_id BIGINT, + machine_id BIGINT, + thread_id BIGINT, + comm_id BIGINT, + dso_id BIGINT, + symbol_id BIGINT, + sym_offset BIGINT, + ip BIGINT, + time BIGINT, + cpu INTEGER, + to_dso_id BIGINT, + to_symbol_id BIGINT, + to_sym_offset BIGINT, + to_ip BIGINT, + period BIGINT, + weight BIGINT, + transaction_ BIGINT, + data_src BIGINT, + branch_type INTEGER, + in_tx BOOLEAN, + call_path_id BIGINT, + insn_count BIGINT, + cyc_count BIGINT, + flags INTEGER) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS call_paths ( + id INTEGER NOT NULL PRIMARY KEY, + parent_id BIGINT, + symbol_id BIGINT, + ip BIGINT) + """) + + # id == 0 means unknown. It is easier to create records for them than + # replace the zeroes with NULLs + self.con.execute("INSERT OR IGNORE INTO selected_events VALUES (0, 'unknown')") + self.con.execute("INSERT OR IGNORE INTO machines VALUES (0, 0, 'unknown')") + self.con.execute("INSERT OR IGNORE INTO threads VALUES (0, 0, 0, -1, -1)") + self.con.execute("INSERT OR IGNORE INTO comms VALUES (0, 'unknown', 0, 0, 0)") + self.con.execute("INSERT OR IGNORE INTO dsos VALUES (0, 0, 'unknown', 'unknown', '')") + self.con.execute("INSERT OR IGNORE INTO symbols VALUES (0, 0, 0, 0, 0, 'unknown')") + + def get_event_id(self, name: str) -> int: + """Get or create event ID.""" + if name in self.caches['events']: + return self.caches['events'][name] + event_id = self.next_id['event'] + self.con.execute("INSERT INTO selected_events VALUES (?, ?)", + (event_id, name)) + self.caches['events'][name] = event_id + self.next_id['event'] += 1 + return event_id + + def get_thread_id(self, pid: int, tid: int) -> int: + """Get or create thread ID.""" + key = (pid, tid) + if key in self.caches['threads']: + return self.caches['threads'][key] + thread_id = self.next_id['thread'] + self.con.execute("INSERT INTO threads VALUES (?, ?, ?, ?, ?)", + (thread_id, 0, pid, pid, tid)) + self.caches['threads'][key] = thread_id + self.next_id['thread'] += 1 + return thread_id + + def get_comm_id(self, comm: str, thread_id: int) -> int: + """Get or create comm ID.""" + if comm in self.caches['comms']: + return self.caches['comms'][comm] + comm_id = self.next_id['comm'] + self.con.execute("INSERT INTO comms VALUES (?, ?, ?, ?, ?)", + (comm_id, comm, thread_id, 0, 0)) + self.con.execute("INSERT INTO comm_threads VALUES (?, ?, ?)", + (comm_id, comm_id, thread_id)) + self.caches['comms'][comm] = comm_id + self.next_id['comm'] += 1 + return comm_id + + def get_dso_id(self, short_name: str, long_name: str, + build_id: str) -> int: + """Get or create DSO ID.""" + if short_name in self.caches['dsos']: + return self.caches['dsos'][short_name] + dso_id = self.next_id['dso'] + self.con.execute("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)", + (dso_id, 0, short_name, long_name, build_id)) + self.caches['dsos'][short_name] = dso_id + self.next_id['dso'] += 1 + return dso_id + + def get_symbol_id(self, dso_id: int, name: str, start: int, + end: int) -> int: + """Get or create symbol ID.""" + key = (dso_id, name) + if key in self.caches['symbols']: + return self.caches['symbols'][key] + symbol_id = self.next_id['symbol'] + self.con.execute("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)", + (symbol_id, dso_id, start, end, 0, name)) + self.caches['symbols'][key] = symbol_id + self.next_id['symbol'] += 1 + return symbol_id + + def get_call_path_id(self, parent_id: int, symbol_id: int, + ip: int) -> int: + """Get or create call path ID.""" + key = (parent_id, symbol_id, ip) + if key in self.caches['call_paths']: + return self.caches['call_paths'][key] + call_path_id = self.next_id['call_path'] + self.con.execute("INSERT INTO call_paths VALUES (?, ?, ?, ?)", + (call_path_id, parent_id, symbol_id, ip)) + self.caches['call_paths'][key] = call_path_id + self.next_id['call_path'] += 1 + return call_path_id + + def process_event(self, sample: perf.sample_event) -> None: + """Callback for processing events.""" + thread_id = self.get_thread_id(sample.sample_pid, sample.sample_tid) + + comm = "Unknown_comm" + try: + if self.session is not None: + proc = self.session.process(sample.sample_pid) + if proc: + comm = proc.comm() + except TypeError: + pass + comm_id = self.get_comm_id(comm, thread_id) + + dso_id = self.get_dso_id( + getattr(sample, 'dso', "Unknown_dso"), + getattr(sample, 'dso_long_name', "Unknown_dso_long"), + getattr(sample, 'dso_bid', "") + ) + + symbol_id = self.get_symbol_id( + dso_id, + getattr(sample, 'symbol', "Unknown_symbol"), + getattr(sample, 'sym_start', 0), + getattr(sample, 'sym_end', 0) + ) + + # Handle callchain + call_path_id = 0 + if hasattr(sample, 'callchain') and sample.callchain: + parent_id = 0 + for node in sample.callchain: + node_dso_id = self.get_dso_id( + node.dso if node.dso else "Unknown_dso", + node.dso if node.dso else "Unknown_dso", + "" + ) + node_symbol_id = self.get_symbol_id( + node_dso_id, + node.symbol if node.symbol else "Unknown_symbol", + 0, 0 + ) + parent_id = self.get_call_path_id(parent_id, node_symbol_id, + node.ip) + call_path_id = parent_id + + # Insert sample + self.con.execute(""" + INSERT INTO samples VALUES ( + NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + """, ( + self.get_event_id(str(sample.evsel)), 0, thread_id, comm_id, + dso_id, symbol_id, getattr(sample, 'sym_offset', 0), + sample.ip, sample.time, sample.cpu, + 0, 0, 0, 0, # to_dso, to_symbol, to_sym_offset, to_ip + getattr(sample, 'period', 0), + getattr(sample, 'weight', 0), + getattr(sample, 'transaction_', 0), + getattr(sample, 'data_src', 0), + 0, # branch_type + getattr(sample, 'in_tx', 0), + call_path_id, + getattr(sample, 'insn_count', 0), + getattr(sample, 'cyc_count', 0), + getattr(sample, 'flags', 0) + )) + + def commit(self) -> None: + """Commit transaction.""" + self.con.commit() + + def close(self) -> None: + """Close connection.""" + self.con.close() + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a sqlite3 database") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("-o", "--output", default="perf.db", + help="Output database name") + args = ap.parse_args() + + if os.path.exists(args.output): + print(f"Error: {args.output} already exists") + sys.exit(1) + + exporter = DatabaseExporter(args.output) + + session = None + try: + session = perf.session(perf.data(args.input), + sample=exporter.process_event) + exporter.session = session + session.process_events() + exporter.commit() + print(f"Successfully exported to {args.output}") + except Exception as e: + print(f"Error processing events: {e}") + if os.path.exists(args.output): + os.remove(args.output) + finally: + exporter.close()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 36/58] perf export-to-sqlite: Port export-to-sqlite to use python module
Sashiko AI review found 8 potential issue(s): - [Critical] AttributeError crash on `sample.ip`, `sample.time`, and `sample.cpu` access. - [High] Silent data loss of branch destinations, periods, and…
Sashiko AI review found 8 potential issue(s):
- [Critical] AttributeError crash on sample.ip, sample.time, and sample.cpu access.
- [High] Silent data loss of branch destinations, periods, and weights.
- [High] Missing SQL views breaking downstream tools and documentation.
- [High] Regression in support for power management events, calls, and context switches.
- [Medium] evsel event names are improperly formatted with 'evsel()' prefix.
- [Medium] getattr fallback behavior results in NULL database values for dso and symbol.
- [Low] Database file removed before connection is closed during error handling.
- [Low] Time-of-Check to Time-of-Use (TOCTOU) race condition when checking for existing database. --
commit f1bba9aa8219f55def547da60006157b32630634
Author: Ian Rogers <[email protected]>
perf export-to-sqlite: Port export-to-sqlite to use python module
This commit ports the export-to-sqlite script to use the modern perf Python module API and the standard library sqlite3. It implements lazy population of lookup tables and adds callchain support.
Quoted 9 lines
tools/perf/python/export-to-sqlite.py new file +3
-0@@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""
[ ... ]
tools/perf/python/export-to-sqlite.py line 19
Quoted 3 lines
tools/perf/python/export-to-sqlite.py new file +3
-0+ $ sqlite3 pt_example + sqlite> .header on + sqlite> select * from samples_view where id < 10;
Is this example still valid? The script no longer contains CREATE VIEW statements for samples_view, which might break downstream tools like exported-sql-viewer.py that rely on these views to function.
Also, this script appears to drop support for tracking call and return elapsed times, context switches, and power management events like ptwrite or cbr. Are these features planned for a future follow-up?
[ ... ]
Quoted 3 lines
tools/perf/python/export-to-sqlite.py new file +3
-0+ def process_event(self, sample: perf.sample_event) -> None: + """Callback for processing events.""" + thread_id = self.get_thread_id(sample.sample_pid, sample.sample_tid)
[ ... ]
tools/perf/python/export-to-sqlite.py line 264
Quoted 5 lines
tools/perf/python/export-to-sqlite.py new file +5
-0+ dso_id = self.get_dso_id( + getattr(sample, 'dso', "Unknown_dso"), + getattr(sample, 'dso_long_name', "Unknown_dso_long"), + getattr(sample, 'dso_bid', "") + )
Does this fallback logic work as expected? If pyrf_sample_event__get_dso returns None instead of raising an AttributeError, getattr will return None rather than the Unknown_dso string. This could lead to inserting NULL into the database instead of the expected textual fallback.
[ ... ]
tools/perf/python/export-to-sqlite.py line 298
Quoted 7 lines
tools/perf/python/export-to-sqlite.py new file +7
-0+ # Insert sample + self.con.execute(""" + INSERT INTO samples VALUES ( + NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + """, ( + self.get_event_id(str(sample.evsel)), 0, thread_id, comm_id,
Will str(sample.evsel) record the event name accurately? The string representation of an evsel object typically returns a formatted string like evsel(cycles). Should this use sample.evsel.name instead?
tools/perf/python/export-to-sqlite.py line 300
Quoted 2 lines
tools/perf/python/export-to-sqlite.py new file +2
-0+ dso_id, symbol_id, getattr(sample, 'sym_offset', 0), + sample.ip, sample.time, sample.cpu,
Can accessing sample.ip, sample.time, and sample.cpu raise an AttributeError here? The perf Python API exposes these properties with a sample_ prefix, such as sample_ip, sample_time, and sample_cpu.
tools/perf/python/export-to-sqlite.py line 306
Quoted 6 lines
tools/perf/python/export-to-sqlite.py new file +6
-0+ 0, 0, 0, 0, # to_dso, to_symbol, to_sym_offset, to_ip + getattr(sample, 'period', 0), + getattr(sample, 'weight', 0), + getattr(sample, 'transaction_', 0), + getattr(sample, 'data_src', 0), + 0, # branch_type
Does this logic intentionally drop branch destination fields and branch_type by hardcoding them to 0? This might break hardware branch tracing use-cases like Intel PT.
Additionally, will getattr(sample, 'period', 0) and getattr(sample, 'weight', 0) successfully retrieve the period and weight? The underlying C-extension attributes are named sample_period and sample_weight, which means getattr might silently fall back to 0 and lose this data.
[ ... ]
Quoted 4 lines
tools/perf/python/export-to-sqlite.py new file +4
-0+if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a sqlite3 database") + ap.add_argument("-i", "--input", default="perf.data",
tools/perf/python/export-to-sqlite.py line 336
Quoted 10 lines
tools/perf/python/export-to-sqlite.py new file +10
-0+ help="Input file name") + ap.add_argument("-o", "--output", default="perf.db", + help="Output database name") + args = ap.parse_args() + + if os.path.exists(args.output): + print(f"Error: {args.output} already exists") + sys.exit(1) + + exporter = DatabaseExporter(args.output)
Is there a race condition here between checking if the file exists and creating the database connection? If a symbolic link is created at the output path right after the check, it might be followed during initialization.
[ ... ]
tools/perf/python/export-to-sqlite.py line 351
Quoted 6 lines
tools/perf/python/export-to-sqlite.py new file +6
-0+ except Exception as e: + print(f"Error processing events: {e}") + if os.path.exists(args.output): + os.remove(args.output) + finally: + exporter.close()
Can removing the database file before calling exporter.close() cause issues? Deleting the main SQLite database file while the connection is still active could lead to permissions errors on some operating systems or leave behind dangling auxiliary files like rollback journals.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=36
Ian Rogers
reviewerPATCH 37/58 perf export-to-postgresql: Port export-to-postgresql to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/export-to-postgresql.py to use the perf Python module API.
Key changes:
- Removed PySide2 dependency by using libpq via ctypes for all DB operations (DDL and COPY).
- Kept the high-performance binary file generation and COPY FROM STDIN approach.
- Implemented lazy population of lookup tables from sample data.
--- tools/perf/python/export-to-postgresql.py | 666 ++++++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100755 tools/perf/python/export-to-postgresql.py
tools/perf/python/export-to-postgresql.py new file +666 -0
@@ -0,0 +1,666 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +r""" +Export perf data to a postgresql database. + +This script has been ported to use the modern perf Python module and +libpq via ctypes. It no longer requires PySide2 or QtSql for exporting. + +The script assumes postgresql is running on the local machine and that the +user has postgresql permissions to create databases. + +An example of using this script with Intel PT: + + $ perf record -e intel_pt//u ls + $ python tools/perf/python/export-to-postgresql.py -i perf.data -o pt_example + +To browse the database, psql can be used e.g. + + $ psql pt_example + pt_example=# select * from samples_view where id < 100; + pt_example=# \d+ + pt_example=# \d+ samples_view + pt_example=# \q + +An example of using the database is provided by the script +exported-sql-viewer.py. Refer to that script for details. + +Tables: + + The tables largely correspond to perf tools' data structures. They are + largely self-explanatory. + + samples + 'samples' is the main table. It represents what instruction was + executing at a point in time when something (a selected event) + happened. The memory address is the instruction pointer or 'ip'. + + branch_types + 'branch_types' provides descriptions for each type of branch. + + comm_threads + 'comm_threads' shows how 'comms' relates to 'threads'. + + comms + 'comms' contains a record for each 'comm' - the name given to the + executable that is running. + + dsos + 'dsos' contains a record for each executable file or library. + + machines + 'machines' can be used to distinguish virtual machines if + virtualization is supported. + + selected_events + 'selected_events' contains a record for each kind of event that + has been sampled. + + symbols + 'symbols' contains a record for each symbol. Only symbols that + have samples are present. + + threads + 'threads' contains a record for each thread. + +Views: + + Most of the tables have views for more friendly display. The views are: + + comm_threads_view + dsos_view + machines_view + samples_view + symbols_view + threads_view + +Ported from tools/perf/scripts/python/export-to-postgresql.py +""" + +import argparse +from ctypes import CDLL, c_char_p, c_int, c_void_p, c_ubyte +import os +import struct +import sys +from typing import Any, Dict, Optional +import perf + +# Need to access PostgreSQL C library directly to use COPY FROM STDIN +try: + libpq = CDLL("libpq.so.5") +except OSError as e: + print(f"Error loading libpq.so.5: {e}") + print("Please ensure PostgreSQL client library is installed.") + sys.exit(1) + +PQconnectdb = libpq.PQconnectdb +PQconnectdb.restype = c_void_p +PQconnectdb.argtypes = [c_char_p] +PQfinish = libpq.PQfinish +PQfinish.argtypes = [c_void_p] +PQstatus = libpq.PQstatus +PQstatus.restype = c_int +PQstatus.argtypes = [c_void_p] +PQexec = libpq.PQexec +PQexec.restype = c_void_p +PQexec.argtypes = [c_void_p, c_char_p] +PQresultStatus = libpq.PQresultStatus +PQresultStatus.restype = c_int +PQresultStatus.argtypes = [c_void_p] +PQputCopyData = libpq.PQputCopyData +PQputCopyData.restype = c_int +PQputCopyData.argtypes = [c_void_p, c_void_p, c_int] +PQputCopyEnd = libpq.PQputCopyEnd +PQputCopyEnd.restype = c_int +PQputCopyEnd.argtypes = [c_void_p, c_void_p] +PQclear = libpq.PQclear +PQclear.argtypes = [c_void_p] + + +def toserverstr(s: str) -> bytes: + """Convert string to server encoding (UTF-8).""" + return bytes(s, "UTF_8") + + +def toclientstr(s: str) -> bytes: + """Convert string to client encoding (UTF-8).""" + return bytes(s, "UTF_8") + + + +class PostgresExporter: + """Handles PostgreSQL connection and exporting of perf events.""" + + def __init__(self, dbname: str): + self.dbname = dbname + self.conn = None + self.session: Optional[perf.session] = None + self.output_dir_name = os.getcwd() + "/" + dbname + "-perf-data" + + self.file_header = struct.pack("!11sii", b"PGCOPY\n\377\r\n\0", 0, 0) + self.file_trailer = b"\377\377" + + # Caches and counters grouped to reduce instance attributes + self.caches: Dict[str, dict] = { + 'threads': {}, + 'comms': {}, + 'dsos': {}, + 'symbols': {}, + 'events': {}, + 'branch_types': {}, + 'call_paths': {} + } + + self.next_id = { + 'thread': 1, + 'comm': 1, + 'dso': 1, + 'symbol': 1, + 'event': 1, + 'branch_type': 1, + 'call_path': 1 + } + + self.files: Dict[str, Any] = {} + self.unhandled_count = 0 + + def connect(self, db_to_use: str) -> None: + """Connect to database.""" + conn_str = toclientstr(f"dbname = {db_to_use}") + self.conn = PQconnectdb(conn_str) + if PQstatus(self.conn) != 0: + raise RuntimeError(f"PQconnectdb failed for {db_to_use}") + + def disconnect(self) -> None: + """Disconnect from database.""" + if self.conn: + PQfinish(self.conn) + self.conn = None + + def do_query(self, sql: str) -> None: + """Execute a query and check status.""" + res = PQexec(self.conn, toserverstr(sql)) + status = PQresultStatus(res) + PQclear(res) + if status not in (1, 2): # PGRES_COMMAND_OK, PGRES_TUPLES_OK + raise RuntimeError(f"Query failed: {sql}") + + + def open_output_file(self, file_name: str): + """Open intermediate binary file.""" + path_name = self.output_dir_name + "/" + file_name + f = open(path_name, "wb+") + f.write(self.file_header) + return f + + def close_output_file(self, f): + """Close intermediate binary file.""" + f.write(self.file_trailer) + f.close() + + def copy_output_file(self, f, table_name: str): + """Copy intermediate file to database.""" + f.seek(0) + sql = f"COPY {table_name} FROM STDIN (FORMAT 'binary')" + res = PQexec(self.conn, toserverstr(sql)) + if PQresultStatus(res) != 4: # PGRES_COPY_IN + PQclear(res) + raise RuntimeError(f"COPY FROM STDIN PQexec failed for {table_name}") + PQclear(res) + + f.seek(0) + data = f.read(65536) + while len(data) > 0: + c_data = (c_ubyte * len(data)).from_buffer_copy(data) + ret = PQputCopyData(self.conn, c_data, len(data)) + if ret != 1: + raise RuntimeError(f"PQputCopyData failed for {table_name}") + data = f.read(65536) + + ret = PQputCopyEnd(self.conn, None) + if ret != 1: + raise RuntimeError(f"PQputCopyEnd failed for {table_name}") + + + + def setup_db(self) -> None: + """Create database and tables. MUST be called after init.""" + os.mkdir(self.output_dir_name) + + self.connect('postgres') + try: + self.do_query(f"CREATE DATABASE {self.dbname}") + except Exception as e: + os.rmdir(self.output_dir_name) + raise e + self.disconnect() + + self.connect(self.dbname) + self.do_query("SET client_min_messages TO WARNING") + + self.do_query(""" + CREATE TABLE selected_events ( + id bigint NOT NULL, + name varchar(80)) + """) + self.do_query(""" + CREATE TABLE machines ( + id bigint NOT NULL, + pid integer, + root_dir varchar(4096)) + """) + self.do_query(""" + CREATE TABLE threads ( + id bigint NOT NULL, + machine_id bigint, + process_id bigint, + pid integer, + tid integer) + """) + self.do_query(""" + CREATE TABLE comms ( + id bigint NOT NULL, + comm varchar(16), + c_thread_id bigint, + c_time bigint, + exec_flag boolean) + """) + self.do_query(""" + CREATE TABLE comm_threads ( + id bigint NOT NULL, + comm_id bigint, + thread_id bigint) + """) + self.do_query(""" + CREATE TABLE dsos ( + id bigint NOT NULL, + machine_id bigint, + short_name varchar(256), + long_name varchar(4096), + build_id varchar(64)) + """) + self.do_query(""" + CREATE TABLE symbols ( + id bigint NOT NULL, + dso_id bigint, + sym_start bigint, + sym_end bigint, + binding integer, + name varchar(2048)) + """) + self.do_query(""" + CREATE TABLE branch_types ( + id integer NOT NULL, + name varchar(80)) + """) + self.do_query(""" + CREATE TABLE samples ( + id bigint NOT NULL, + evsel_id bigint, + machine_id bigint, + thread_id bigint, + comm_id bigint, + dso_id bigint, + symbol_id bigint, + sym_offset bigint, + ip bigint, + time bigint, + cpu integer, + to_dso_id bigint, + to_symbol_id bigint, + to_sym_offset bigint, + to_ip bigint, + period bigint, + weight bigint, + transaction_ bigint, + data_src bigint, + branch_type integer, + in_tx boolean, + call_path_id bigint, + insn_count bigint, + cyc_count bigint, + flags integer) + """) + self.do_query(""" + CREATE TABLE call_paths ( + id bigint NOT NULL, + parent_id bigint, + symbol_id bigint, + ip bigint) + """) + + self.files['evsel'] = self.open_output_file("evsel_table.bin") + self.files['machine'] = self.open_output_file("machine_table.bin") + self.files['thread'] = self.open_output_file("thread_table.bin") + self.files['comm'] = self.open_output_file("comm_table.bin") + self.files['comm_thread'] = self.open_output_file("comm_thread_table.bin") + self.files['dso'] = self.open_output_file("dso_table.bin") + self.files['symbol'] = self.open_output_file("symbol_table.bin") + self.files['branch_type'] = self.open_output_file("branch_type_table.bin") + self.files['sample'] = self.open_output_file("sample_table.bin") + self.files['call_path'] = self.open_output_file("call_path_table.bin") + + self.write_evsel(0, "unknown") + self.write_machine(0, 0, "unknown") + self.write_thread(0, 0, 0, -1, -1) + self.write_comm(0, "unknown", 0, 0, 0) + self.write_dso(0, 0, "unknown", "unknown", "") + self.write_symbol(0, 0, 0, 0, 0, "unknown") + self.write_call_path(0, 0, 0, 0) + + def write_evsel(self, evsel_id: int, name: str) -> None: + """Write event to binary file.""" + name_bytes = toserverstr(name) + n = len(name_bytes) + fmt = "!hiqi" + str(n) + "s" + value = struct.pack(fmt, 2, 8, evsel_id, n, name_bytes) + self.files['evsel'].write(value) + + def write_machine(self, machine_id: int, pid: int, root_dir: str) -> None: + """Write machine to binary file.""" + rd_bytes = toserverstr(root_dir) + n = len(rd_bytes) + fmt = "!hiqiii" + str(n) + "s" + value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, rd_bytes) + self.files['machine'].write(value) + + + def write_thread(self, thread_id: int, machine_id: int, process_id: int, + pid: int, tid: int) -> None: + """Write thread to binary file.""" + value = struct.pack("!hiqiqiqiiii", 5, 8, thread_id, 8, machine_id, + 8, process_id, 4, pid, 4, tid) + self.files['thread'].write(value) + + + def write_comm(self, comm_id: int, comm_str: str, thread_id: int, + time: int, exec_flag: int) -> None: + """Write comm to binary file.""" + comm_bytes = toserverstr(comm_str) + n = len(comm_bytes) + fmt = "!hiqi" + str(n) + "s" + "iqiqiB" + value = struct.pack(fmt, 5, 8, comm_id, n, comm_bytes, 8, + thread_id, 8, time, 1, exec_flag) + self.files['comm'].write(value) + + def write_comm_thread(self, comm_thread_id: int, comm_id: int, + thread_id: int) -> None: + """Write comm_thread to binary file.""" + fmt = "!hiqiqiq" + value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id) + self.files['comm_thread'].write(value) + + + def write_dso(self, dso_id: int, machine_id: int, short_name: str, + long_name: str, build_id: str) -> None: + """Write DSO to binary file.""" + sn_bytes = toserverstr(short_name) + ln_bytes = toserverstr(long_name) + bi_bytes = toserverstr(build_id) + n1, n2, n3 = len(sn_bytes), len(ln_bytes), len(bi_bytes) + fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s" + value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1, + sn_bytes, n2, ln_bytes, n3, bi_bytes) + self.files['dso'].write(value) + + + def write_symbol(self, symbol_id: int, dso_id: int, sym_start: int, + sym_end: int, binding: int, symbol_name: str) -> None: + """Write symbol to binary file.""" + name_bytes = toserverstr(symbol_name) + n = len(name_bytes) + fmt = "!hiqiqiqiqiii" + str(n) + "s" + value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, + sym_start, 8, sym_end, 4, binding, n, name_bytes) + self.files['symbol'].write(value) + + def write_call_path(self, cp_id: int, parent_id: int, symbol_id: int, + ip: int) -> None: + """Write call path to binary file.""" + fmt = "!hiqiqiqiq" + value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip) + self.files['call_path'].write(value) + + + def write_sample(self, sample_id: int, evsel_id: int, thread_id: int, + comm_id: int, dso_id: int, symbol_id: int, + sample: perf.sample_event, call_path_id: int) -> None: + """Write sample to binary file.""" + value = struct.pack( + "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiBiqiqiqii", + 25, 8, sample_id, 8, evsel_id, 8, 0, 8, thread_id, 8, comm_id, + 8, dso_id, 8, symbol_id, 8, getattr(sample, 'sym_offset', 0), + 8, sample.ip, 8, sample.time, 4, sample.cpu, + 8, 0, 8, 0, 8, 0, 8, 0, + getattr(sample, 'period', 0), + getattr(sample, 'weight', 0), + getattr(sample, 'transaction_', 0), + getattr(sample, 'data_src', 0), + 0, + getattr(sample, 'in_tx', 0), + call_path_id, + getattr(sample, 'insn_count', 0), + getattr(sample, 'cyc_count', 0), + 4, getattr(sample, 'flags', 0) + ) + self.files['sample'].write(value) + + def get_event_id(self, name: str) -> int: + """Get or create event ID.""" + if name in self.caches['events']: + return self.caches['events'][name] + event_id = self.next_id['event'] + self.write_evsel(event_id, name) + self.caches['events'][name] = event_id + self.next_id['event'] += 1 + return event_id + + def get_thread_id(self, pid: int, tid: int) -> int: + """Get or create thread ID.""" + key = (pid, tid) + if key in self.caches['threads']: + return self.caches['threads'][key] + thread_id = self.next_id['thread'] + self.write_thread(thread_id, 0, pid, pid, tid) + self.caches['threads'][key] = thread_id + self.next_id['thread'] += 1 + return thread_id + + def get_comm_id(self, comm: str, thread_id: int) -> int: + """Get or create comm ID.""" + if comm in self.caches['comms']: + return self.caches['comms'][comm] + comm_id = self.next_id['comm'] + self.write_comm(comm_id, comm, thread_id, 0, 0) + self.write_comm_thread(comm_id, comm_id, thread_id) + self.caches['comms'][comm] = comm_id + self.next_id['comm'] += 1 + return comm_id + + def get_dso_id(self, short_name: str, long_name: str, + build_id: str) -> int: + """Get or create DSO ID.""" + if short_name in self.caches['dsos']: + return self.caches['dsos'][short_name] + dso_id = self.next_id['dso'] + self.write_dso(dso_id, 0, short_name, long_name, build_id) + self.caches['dsos'][short_name] = dso_id + self.next_id['dso'] += 1 + return dso_id + + def get_symbol_id(self, dso_id: int, name: str, start: int, + end: int) -> int: + """Get or create symbol ID.""" + key = (dso_id, name) + if key in self.caches['symbols']: + return self.caches['symbols'][key] + symbol_id = self.next_id['symbol'] + self.write_symbol(symbol_id, dso_id, start, end, 0, name) + self.caches['symbols'][key] = symbol_id + self.next_id['symbol'] += 1 + return symbol_id + + def get_call_path_id(self, parent_id: int, symbol_id: int, + ip: int) -> int: + """Get or create call path ID.""" + key = (parent_id, symbol_id, ip) + if key in self.caches['call_paths']: + return self.caches['call_paths'][key] + call_path_id = self.next_id['call_path'] + self.write_call_path(call_path_id, parent_id, symbol_id, ip) + self.caches['call_paths'][key] = call_path_id + self.next_id['call_path'] += 1 + return call_path_id + + def process_event(self, sample: perf.sample_event) -> None: + """Callback for processing events.""" + thread_id = self.get_thread_id(sample.sample_pid, sample.sample_tid) + + comm = "Unknown_comm" + try: + if self.session is not None: + proc = self.session.process(sample.sample_pid) + if proc: + comm = proc.comm() + except TypeError: + pass + comm_id = self.get_comm_id(comm, thread_id) + + dso_id = self.get_dso_id( + getattr(sample, 'dso', "Unknown_dso"), + getattr(sample, 'dso_long_name', "Unknown_dso_long"), + getattr(sample, 'dso_bid', "") + ) + + symbol_id = self.get_symbol_id( + dso_id, + getattr(sample, 'symbol', "Unknown_symbol"), + getattr(sample, 'sym_start', 0), + getattr(sample, 'sym_end', 0) + ) + + call_path_id = 0 + if hasattr(sample, 'callchain') and sample.callchain: + parent_id = 0 + for node in sample.callchain: + node_dso_id = self.get_dso_id( + node.dso if node.dso else "Unknown_dso", + node.dso if node.dso else "Unknown_dso", + "" + ) + node_symbol_id = self.get_symbol_id( + node_dso_id, + node.symbol if node.symbol else "Unknown_symbol", + 0, 0 + ) + parent_id = self.get_call_path_id(parent_id, node_symbol_id, + node.ip) + call_path_id = parent_id + + sample_id = self.next_id['event'] + self.write_sample(sample_id, self.get_event_id(str(sample.evsel)), + thread_id, comm_id, dso_id, symbol_id, sample, + call_path_id) + self.next_id['event'] += 1 + + def finalize(self) -> None: + """Copy files to database and add keys/views.""" + print("Copying to database...") + for name, f in self.files.items(): + self.close_output_file(f) + table_name = name + "s" if name != "call_path" else "call_paths" + if name == "evsel": + table_name = "selected_events" + self.copy_output_file(f, table_name) + + print("Removing intermediate files...") + for name, f in self.files.items(): + os.unlink(f.name) + os.rmdir(self.output_dir_name) + + print("Adding primary keys") + self.do_query("ALTER TABLE selected_events ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE machines ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE threads ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE comms ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE comm_threads ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE dsos ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE symbols ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE branch_types ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE samples ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE call_paths ADD PRIMARY KEY (id)") + + print("Creating views...") + self.do_query(""" + CREATE VIEW machines_view AS + SELECT id, pid, root_dir, + CASE WHEN id=0 THEN 'unknown' WHEN pid=-1 THEN 'host' ELSE 'guest' END AS host_or_guest + FROM machines + """) + self.do_query(""" + CREATE VIEW dsos_view AS + SELECT id, machine_id, + (SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, + short_name, long_name, build_id + FROM dsos + """) + self.do_query(""" + CREATE VIEW symbols_view AS + SELECT id, name, + (SELECT short_name FROM dsos WHERE id=dso_id) AS dso, + dso_id, sym_start, sym_end, + CASE WHEN binding=0 THEN 'local' WHEN binding=1 THEN 'global' ELSE 'weak' END AS binding + FROM symbols + """) + self.do_query(""" + CREATE VIEW threads_view AS + SELECT id, machine_id, + (SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, + process_id, pid, tid + FROM threads + """) + self.do_query(""" + CREATE VIEW samples_view AS + SELECT id, time, cpu, + (SELECT pid FROM threads WHERE id = thread_id) AS pid, + (SELECT tid FROM threads WHERE id = thread_id) AS tid, + (SELECT comm FROM comms WHERE id = comm_id) AS command, + (SELECT name FROM selected_events WHERE id = evsel_id) AS event, + to_hex(ip) AS ip_hex, + (SELECT name FROM symbols WHERE id = symbol_id) AS symbol, + sym_offset, + (SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name, + to_hex(to_ip) AS to_ip_hex, + (SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol, + to_sym_offset, + (SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name, + (SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name, + in_tx, insn_count, cyc_count, flags + FROM samples + """) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a postgresql database") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("-o", "--output", required=True, + help="Output database name") + args = ap.parse_args() + + exporter = PostgresExporter(args.output) + exporter.setup_db() + + session = None + try: + session = perf.session(perf.data(args.input), + sample=exporter.process_event) + exporter.session = session + session.process_events() + exporter.finalize() + print(f"Successfully exported to {args.output}") + except Exception as e: + print(f"Error processing events: {e}") + finally: + exporter.disconnect()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 37/58] perf export-to-postgresql: Port export-to-postgresql to use python module
Sashiko AI review found 8 potential issue(s): - [High] Immediate runtime crash in `write_sample` due to `struct.pack` argument mismatch. - [High] `finalize()` attempts to read from closed files…
Sashiko AI review found 8 potential issue(s):
- [High] Immediate runtime crash in write_sample due to struct.pack argument mismatch.
- [High] finalize() attempts to read from closed files, causing an I/O error.
- [High] Missing PQgetResult after PQputCopyEnd leaves the PostgreSQL connection in a desynchronized state.
- [High] Caching command IDs solely by name drops relationships for different threads executing the same command.
- [Medium] Temporary output directories and files are permanently leaked on errors.
- [Medium] Unquoted database name in CREATE DATABASE query causes syntax errors.
- [Low] The branch_types lookup table is completely unpopulated.
- [Low] Hardcoding the specific shared library version 'libpq.so.5' reduces portability.
--
commit b6c1a19bed966a424b4379f69e4ee67f64608381
Author: Ian Rogers <[email protected]>
perf export-to-postgresql: Port export-to-postgresql to use python module
This commit ports export-to-postgresql.py to use the perf Python module API instead of PySide2. It uses libpq via ctypes for database operations while keeping the high-performance binary file generation approach. Lazy population of lookup tables from sample data is also implemented.
Quoted 6 lines
tools/perf/python/export-to-postgresql.py new file +0
-0@@ -0,0 +1,666 @@
[ ... ]
tools/perf/python/export-to-postgresql.py line 90
Quoted 11 lines
tools/perf/python/export-to-postgresql.py new file +11
-0+import argparse +from ctypes import CDLL, c_char_p, c_int, c_void_p, c_ubyte +import os +import struct +import sys +from typing import Any, Dict, Optional +import perf + +# Need to access PostgreSQL C library directly to use COPY FROM STDIN +try: + libpq = CDLL("libpq.so.5")
Is hardcoding "libpq.so.5" strictly required here?
On systems where the PostgreSQL client library uses a different naming convention (e.g., libpq.so or libpq.so.6), this might fail to load. Could we use ctypes.util.find_library('pq') to locate the library dynamically?
[ ... ]
Quoted 12 lines
tools/perf/python/export-to-postgresql.py new file +12
-0+ def copy_output_file(self, f, table_name: str): + """Copy intermediate file to database.""" + f.seek(0) + sql = f"COPY {table_name} FROM STDIN (FORMAT 'binary')" + res = PQexec(self.conn, toserverstr(sql)) + if PQresultStatus(res) != 4: # PGRES_COPY_IN + PQclear(res) + raise RuntimeError(f"COPY FROM STDIN PQexec failed for {table_name}") + PQclear(res) + + f.seek(0) + data = f.read(65536)
tools/perf/python/export-to-postgresql.py line 222
Quoted 10 lines
tools/perf/python/export-to-postgresql.py new file +10
-0+ while len(data) > 0: + c_data = (c_ubyte * len(data)).from_buffer_copy(data) + ret = PQputCopyData(self.conn, c_data, len(data)) + if ret != 1: + raise RuntimeError(f"PQputCopyData failed for {table_name}") + data = f.read(65536) + + ret = PQputCopyEnd(self.conn, None) + if ret != 1: + raise RuntimeError(f"PQputCopyEnd failed for {table_name}")
After calling PQputCopyEnd(), is it necessary to call PQgetResult() until it returns NULL?
The libpq C-API requires clearing the PGRES_COMMAND_OK status of the COPY command. Failing to do so might leave the database connection in a busy state, causing the next PQexec call (e.g., copying the next table) to fail with an "another command is already in progress" error.
[ ... ]
tools/perf/python/export-to-postgresql.py line 232
Quoted 7 lines
tools/perf/python/export-to-postgresql.py new file +7
-0+ def setup_db(self) -> None: + """Create database and tables. MUST be called after init.""" + os.mkdir(self.output_dir_name) + + self.connect('postgres') + try: + self.do_query(f"CREATE DATABASE {self.dbname}")
Will this query fail if the user provides a database name with hyphens (e.g., -o my-perf-data)?
PostgreSQL usually requires double-quoted identifiers for hyphens. This might break functionality for common naming conventions or expose the script to SQL injection.
[ ... ]
tools/perf/python/export-to-postgresql.py line 339
Quoted 8 lines
tools/perf/python/export-to-postgresql.py new file +8
-0+ self.files['evsel'] = self.open_output_file("evsel_table.bin") + self.files['machine'] = self.open_output_file("machine_table.bin") + self.files['thread'] = self.open_output_file("thread_table.bin") + self.files['comm'] = self.open_output_file("comm_table.bin") + self.files['comm_thread'] = self.open_output_file("comm_thread_table.bin") + self.files['dso'] = self.open_output_file("dso_table.bin") + self.files['symbol'] = self.open_output_file("symbol_table.bin") + self.files['branch_type'] = self.open_output_file("branch_type_table.bin")
The script initializes a binary file for the branch_types table, but is there logic to populate it?
It appears the table is completely unpopulated. Will queries relying on joining against branch_types to map branch names consistently return NULL?
[ ... ]
Quoted 11 lines
tools/perf/python/export-to-postgresql.py new file +11
-0+ def write_sample(self, sample_id: int, evsel_id: int, thread_id: int, + comm_id: int, dso_id: int, symbol_id: int, + sample: perf.sample_event, call_path_id: int) -> None: + """Write sample to binary file.""" + value = struct.pack( + "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiBiqiqiqii", + 25, 8, sample_id, 8, evsel_id, 8, 0, 8, thread_id, 8, comm_id, + 8, dso_id, 8, symbol_id, 8, getattr(sample, 'sym_offset', 0), + 8, sample.ip, 8, sample.time, 4, sample.cpu, + 8, 0, 8, 0, 8, 0, 8, 0, + getattr(sample, 'period', 0),
tools/perf/python/export-to-postgresql.py line 445
Quoted 10 lines
tools/perf/python/export-to-postgresql.py new file +10
-0+ getattr(sample, 'weight', 0), + getattr(sample, 'transaction_', 0), + getattr(sample, 'data_src', 0), + 0, + getattr(sample, 'in_tx', 0), + call_path_id, + getattr(sample, 'insn_count', 0), + getattr(sample, 'cyc_count', 0), + 4, getattr(sample, 'flags', 0) + )
Does this struct.pack format string match the number of provided arguments?
The format string requires 51 arguments to properly serialize the data into PostgreSQL's binary COPY format, but starting around the period attribute, the length prefix arguments seem to be omitted. Since only 42 arguments are provided, won't this unconditionally raise a struct.error exception on the first processed event?
[ ... ]
tools/perf/python/export-to-postgresql.py line 478
Quoted 10 lines
tools/perf/python/export-to-postgresql.py new file +10
-0+ def get_comm_id(self, comm: str, thread_id: int) -> int: + """Get or create comm ID.""" + if comm in self.caches['comms']: + return self.caches['comms'][comm] + comm_id = self.next_id['comm'] + self.write_comm(comm_id, comm, thread_id, 0, 0) + self.write_comm_thread(comm_id, comm_id, thread_id) + self.caches['comms'][comm] = comm_id + self.next_id['comm'] += 1 + return comm_id
If a second thread executes the same command name, the cache lookup succeeds and bypasses the write_comm_thread() call. Does this mean no comm_thread relationship is written for the new thread?
This might silently omit associations for any thread that happens to share a command name with an earlier thread.
[ ... ]
tools/perf/python/export-to-postgresql.py line 574
Quoted 9 lines
tools/perf/python/export-to-postgresql.py new file +9
-0+ def finalize(self) -> None: + """Copy files to database and add keys/views.""" + print("Copying to database...") + for name, f in self.files.items(): + self.close_output_file(f) + table_name = name + "s" if name != "call_path" else "call_paths" + if name == "evsel": + table_name = "selected_events" + self.copy_output_file(f, table_name)
Here finalize() calls self.close_output_file(f) which closes the file, and then immediately passes it to self.copy_output_file().
Since copy_output_file() attempts to execute f.seek(0) and f.read(65536), will this raise a ValueError for attempting I/O operations on a closed file?
[ ... ]
Quoted 14 lines
tools/perf/python/export-to-postgresql.py new file +14
-0+if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a postgresql database") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("-o", "--output", required=True, + help="Output database name") + args = ap.parse_args() + + exporter = PostgresExporter(args.output) + exporter.setup_db() + + session = None + try:
tools/perf/python/export-to-postgresql.py line 666
Quoted 10 lines
tools/perf/python/export-to-postgresql.py new file +10
-0+ session = perf.session(perf.data(args.input), + sample=exporter.process_event) + exporter.session = session + session.process_events() + exporter.finalize() + print(f"Successfully exported to {args.output}") + except Exception as e: + print(f"Error processing events: {e}") + finally: + exporter.disconnect()
If an exception occurs during setup_db() or process_events(), the finalize() method is bypassed.
Could this leak the temporary directory and binary trace files, causing subsequent runs to fail with a FileExistsError on os.mkdir()?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=37
Ian Rogers
reviewerPATCH 38/58 perf failed-syscalls-by-pid: Port failed-syscalls-by-pid to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/failed-syscalls-by-pid.py to use the perf Python module API.
Key changes:
- Used perf.syscall_name() to resolve syscall names instead of legacy Util library.
- Used standard collections.defaultdict for nested statistics aggregation.
- Used errno.errorcode for resolving error strings.
- Supported optional filtering by COMM or PID via command line arguments.
--- tools/perf/python/failed-syscalls-by-pid.py | 121 ++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100755 tools/perf/python/failed-syscalls-by-pid.py
tools/perf/python/failed-syscalls-by-pid.py new file +121 -0
@@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Displays system-wide failed system call totals, broken down by pid. +If a [comm] or [pid] arg is specified, only syscalls called by it are displayed. + +Ported from tools/perf/scripts/python/failed-syscalls-by-pid.py +""" + +import argparse +from collections import defaultdict +import errno +from typing import DefaultDict, Optional +import perf + +# Type alias to avoid long lines +SyscallDict = DefaultDict[str, DefaultDict[int, DefaultDict[int, DefaultDict[int, int]]]] + + +def strerror(nr: int) -> str: + """Return error string for a given errno.""" + try: + return errno.errorcode[abs(nr)] + except KeyError: + return f"Unknown {nr} errno" + + +class SyscallAnalyzer: + """Analyzes failed syscalls and aggregates counts.""" + + def __init__(self, for_comm: Optional[str] = None, for_pid: Optional[int] = None): + self.for_comm = for_comm + self.for_pid = for_pid + # Structure: syscalls[comm][pid][syscall_id][ret] = count + self.syscalls: SyscallDict = defaultdict( + lambda: defaultdict( + lambda: defaultdict( + lambda: defaultdict(int) + ) + ) + ) + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + if str(sample.evsel) not in ("evsel(raw_syscalls:sys_exit)", "evsel(syscalls:sys_exit)"): + return + + comm = getattr(sample, "comm", "Unknown") + pid = sample.sample_pid + + if self.for_comm and comm != self.for_comm: + return + if self.for_pid and pid != self.for_pid: + return + + ret = getattr(sample, "ret", 0) + if ret < 0: + syscall_id = getattr(sample, "id", -1) + if syscall_id == -1: + # Try to get it from another field name if available + syscall_id = getattr(sample, "sys_id", -1) + + self.syscalls[comm][pid][syscall_id][ret] += 1 + + def print_summary(self) -> None: + """Print aggregated statistics.""" + if self.for_comm is not None: + print(f"\nsyscall errors for {self.for_comm}:\n") + elif self.for_pid is not None: + print(f"\nsyscall errors for PID {self.for_pid}:\n") + else: + print("\nsyscall errors:\n") + + print(f"{'comm [pid]':<30} {'count':>10}") + print(f"{'-' * 30:<30} {'-' * 10:>10}") + + for comm in sorted(self.syscalls.keys()): + for pid in sorted(self.syscalls[comm].keys()): + print(f"\n{comm} [{pid}]") + for syscall_id in sorted(self.syscalls[comm][pid].keys()): + try: + name = perf.syscall_name(syscall_id) + except AttributeError: + name = str(syscall_id) + print(f" syscall: {name:<16}") + + items = self.syscalls[comm][pid][syscall_id].items() + # Sort by count (descending), then by return value + sorted_items = sorted(items, key=lambda kv: (kv[1], kv[0]), reverse=True) + for ret, val in sorted_items: + print(f" err = {strerror(ret):<20} {val:10d}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Displays system-wide failed system call totals, broken down by pid.") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("filter", nargs="?", help="COMM or PID to filter by") + args = ap.parse_args() + + F_COMM = None + F_PID = None + + if args.filter: + try: + F_PID = int(args.filter) + except ValueError: + F_COMM = args.filter + + analyzer = SyscallAnalyzer(F_COMM, F_PID) + + try: + print("Press control+C to stop and show the summary") + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + analyzer.print_summary() + except KeyboardInterrupt: + analyzer.print_summary() + except Exception as e: + print(f"Error processing events: {e}")
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 38/58] perf failed-syscalls-by-pid: Port failed-syscalls-by-pid to use python module
Sashiko AI review found 2 potential issue(s): - [High] The script attempts to extract the process comm using `getattr(sample, 'comm', 'Unknown')`, but the new perf Python module does not inject…
Sashiko AI review found 2 potential issue(s):
- [High] The script attempts to extract the process comm using getattr(sample, 'comm', 'Unknown'), but the new perf Python module does not inject comm into the sample object. This causes all events to be aggregated under 'Unknown', fundamentally breaking the output grouping and causing the --comm filter to drop all events.
- [High] Formatting a potentially None syscall name with a width specifier causes a TypeError crash.
--
commit 9366fa7bfa752a994e4a854653a22852e8debe63
Author: Ian Rogers <[email protected]>
perf failed-syscalls-by-pid: Port failed-syscalls-by-pid to use python module
This commit ports the failed-syscalls-by-pid script to the new perf Python module API. It updates syscall name resolution, statistic aggregation, error string lookup, and adds optional COMM/PID filtering.
Quoted 5 lines
tools/perf/python/failed-syscalls-by-pid.py new file +0
-0
[ ... ]
tools/perf/python/failed-syscalls-by-pid.py line 48
Quoted 6 lines
tools/perf/python/failed-syscalls-by-pid.py new file +6
-0+ def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + if str(sample.evsel) not in ("evsel(raw_syscalls:sys_exit)", "evsel(syscalls:sys_exit)"): + return + + comm = getattr(sample, "comm", "Unknown")
Does the perf.sample_event object expose a comm attribute in the new Python API?
If it is missing, getattr will fall back to "Unknown" for every event. This would cause all events to be aggregated together under a single name, and would drop all events if the comm filter is active.
Should this resolve the comm by querying the session, for example by calling session.process(sample.pid).comm() instead?
[ ... ]
Quoted 2 lines
tools/perf/python/failed-syscalls-by-pid.py new file +2
-0+ def print_summary(self) -> None: + """Print aggregated statistics."""
[ ... ]
tools/perf/python/failed-syscalls-by-pid.py line 85
Quoted 9 lines
tools/perf/python/failed-syscalls-by-pid.py new file +9
-0+ for comm in sorted(self.syscalls.keys()): + for pid in sorted(self.syscalls[comm].keys()): + print(f"\n{comm} [{pid}]") + for syscall_id in sorted(self.syscalls[comm][pid].keys()): + try: + name = perf.syscall_name(syscall_id) + except AttributeError: + name = str(syscall_id) + print(f" syscall: {name:<16}")
If perf.syscall_name() returns None for invalid or unknown syscall IDs (such as the fallback -1), will applying a width format specifier (<16) to a NoneType cause a TypeError and crash the script?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=38
Ian Rogers
reviewerPATCH 39/58 perf intel-pt-events: Port intel-pt-events/libxed to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/.
- Refactored intel-pt-events.py to use a class structure to eliminate global state and improve maintainability.
- Removed Python 2 compatibility checks.
- Renamed methods in libxed.py to snake_case (Instruction -> instruction, SetMode -> set_mode, DisassembleOne -> disassemble_one) to comply with pylint.
--- tools/perf/python/intel-pt-events.py | 421 +++++++++++++++++++++++++++ tools/perf/python/libxed.py | 119 ++++++++ 2 files changed, 540 insertions(+) create mode 100755 tools/perf/python/intel-pt-events.py create mode 100755 tools/perf/python/libxed.py
tools/perf/python/intel-pt-events.py new file +421 -0
@@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Print Intel PT Events including Power Events and PTWRITE. +Ported from tools/perf/scripts/python/intel-pt-events.py +""" + +import argparse +import contextlib +from ctypes import addressof, create_string_buffer +import io +import os +import struct +import sys +from typing import Any, Optional +import perf + +# Try to import LibXED from legacy directory if available in PYTHONPATH +try: + from libxed import LibXED # type: ignore +except ImportError: + LibXED = None + + +class IntelPTAnalyzer: + """Analyzes Intel PT events and prints details.""" + + def __init__(self, cfg: argparse.Namespace): + self.args = cfg + self.insn = False + self.src = False + self.source_file_name: Optional[str] = None + self.line_number: int = 0 + self.dso: Optional[str] = None + self.stash_dict: dict[int, list[str]] = {} + self.output: Any = None + self.output_pos: int = 0 + self.cpu: int = -1 + self.time: int = 0 + self.switch_str: dict[int, str] = {} + + if cfg.insn_trace: + print("Intel PT Instruction Trace") + self.insn = True + elif cfg.src_trace: + print("Intel PT Source Trace") + self.insn = True + self.src = True + else: + print("Intel PT Branch Trace, Power Events, Event Trace and PTWRITE") + + self.disassembler: Any = None + if self.insn and LibXED is not None: + try: + self.disassembler = LibXED() + except Exception as e: + print(f"Failed to initialize LibXED: {e}") + self.disassembler = None + + def print_ptwrite(self, raw_buf: bytes) -> None: + """Print PTWRITE data.""" + data = struct.unpack_from("<IQ", raw_buf) + flags = data[0] + payload = data[1] + exact_ip = flags & 1 + try: + s = payload.to_bytes(8, "little").decode("ascii").rstrip("\x00") + if not s.isprintable(): + s = "" + except (UnicodeDecodeError, ValueError): + s = "" + print(f"IP: {exact_ip} payload: {payload:#x} {s}", end=' ') + + def print_cbr(self, raw_buf: bytes) -> None: + """Print CBR data.""" + data = struct.unpack_from("<BBBBII", raw_buf) + cbr = data[0] + f = (data[4] + 500) // 1000 + p = ((cbr * 1000 // data[2]) + 5) // 10 + print(f"{cbr:3u} freq: {f:4u} MHz ({p:3u}%)", end=' ') + + def print_mwait(self, raw_buf: bytes) -> None: + """Print MWAIT data.""" + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + hints = payload & 0xff + extensions = (payload >> 32) & 0x3 + print(f"hints: {hints:#x} extensions: {extensions:#x}", end=' ') + + def print_pwre(self, raw_buf: bytes) -> None: + """Print PWRE data.""" + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + hw = (payload >> 7) & 1 + cstate = (payload >> 12) & 0xf + subcstate = (payload >> 8) & 0xf + print(f"hw: {hw} cstate: {cstate} sub-cstate: {subcstate}", end=' ') + + def print_exstop(self, raw_buf: bytes) -> None: + """Print EXSTOP data.""" + data = struct.unpack_from("<I", raw_buf) + flags = data[0] + exact_ip = flags & 1 + print(f"IP: {exact_ip}", end=' ') + + def print_pwrx(self, raw_buf: bytes) -> None: + """Print PWRX data.""" + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + deepest_cstate = payload & 0xf + last_cstate = (payload >> 4) & 0xf + wake_reason = (payload >> 8) & 0xf + print(f"deepest cstate: {deepest_cstate} last cstate: {last_cstate} " + f"wake reason: {wake_reason:#x}", end=' ') + + def print_psb(self, raw_buf: bytes) -> None: + """Print PSB data.""" + data = struct.unpack_from("<IQ", raw_buf) + offset = data[1] + print(f"offset: {offset:#x}", end=' ') + + def print_evt(self, raw_buf: bytes) -> None: + """Print EVT data.""" + glb_cfe = ["", "INTR", "IRET", "SMI", "RSM", "SIPI", "INIT", "VMENTRY", "VMEXIT", + "VMEXIT_INTR", "SHUTDOWN", "", "UINT", "UIRET"] + [""] * 18 + glb_evd = ["", "PFA", "VMXQ", "VMXR"] + [""] * 60 + + data = struct.unpack_from("<BBH", raw_buf) + typ = data[0] & 0x1f + ip_flag = (data[0] & 0x80) >> 7 + vector = data[1] + evd_cnt = data[2] + s = glb_cfe[typ] + if s: + print(f" cfe: {s} IP: {ip_flag} vector: {vector}", end=' ') + else: + print(f" cfe: {typ} IP: {ip_flag} vector: {vector}", end=' ') + pos = 4 + for _ in range(evd_cnt): + data = struct.unpack_from("<QQ", raw_buf, pos) + et = data[0] & 0x3f + s = glb_evd[et] + if s: + print(f"{s}: {data[1]:#x}", end=' ') + else: + print(f"EVD_{et}: {data[1]:#x}", end=' ') + pos += 16 + + def print_iflag(self, raw_buf: bytes) -> None: + """Print IFLAG data.""" + data = struct.unpack_from("<IQ", raw_buf) + iflag = data[0] & 1 + old_iflag = iflag ^ 1 + via_branch = data[0] & 2 + s = "via" if via_branch else "non" + print(f"IFLAG: {old_iflag}->{iflag} {s} branch", end=' ') + + def common_start_str(self, comm: str, sample: perf.sample_event) -> str: + """Return common start string for display.""" + ts = sample.time + cpu = sample.cpu + pid = sample.pid + tid = sample.tid + machine_pid = getattr(sample, "machine_pid", 0) + if machine_pid: + vcpu = getattr(sample, "vcpu", -1) + return (f"VM:{machine_pid:5d} VCPU:{vcpu:03d} {comm:>16s} {pid:5u}/{tid:<5u} " + f"[{cpu:03u}] {ts // 1000000000:9u}.{ts % 1000000000:09u} ") + return (f"{comm:>16s} {pid:5u}/{tid:<5u} [{cpu:03u}] " + f"{ts // 1000000000:9u}.{ts % 1000000000:09u} ") + + def print_common_start(self, comm: str, sample: perf.sample_event, name: str) -> None: + """Print common start info.""" + flags_disp = getattr(sample, "flags_disp", "") + print(self.common_start_str(comm, sample) + f"{name:>8s} {flags_disp:>21s}", end=' ') + + def print_instructions_start(self, comm: str, sample: perf.sample_event) -> None: + """Print instructions start info.""" + flags = getattr(sample, "flags_disp", "") + if "x" in flags: + print(self.common_start_str(comm, sample) + "x", end=' ') + else: + print(self.common_start_str(comm, sample), end=' ') + + def disassem(self, insn: bytes, ip: int) -> tuple[int, str]: + """Disassemble instruction using LibXED.""" + inst = self.disassembler.instruction() + self.disassembler.set_mode(inst, 0) # Assume 64-bit + buf = create_string_buffer(64) + buf.value = insn + return self.disassembler.disassemble_one(inst, addressof(buf), len(insn), ip) + + def print_common_ip(self, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print IP and symbol info.""" + ip = sample.ip + offs = f"+{sample.symoff:#x}" if hasattr(sample, "symoff") else "" + cyc_cnt = getattr(sample, "cyc_cnt", 0) + if cyc_cnt: + insn_cnt = getattr(sample, "insn_cnt", 0) + ipc_str = f" IPC: {insn_cnt / cyc_cnt:#.2f} ({insn_cnt}/{cyc_cnt})" + else: + ipc_str = "" + + if self.insn and self.disassembler is not None: + try: + insn = sample.insn() + except AttributeError: + insn = None + if insn: + cnt, text = self.disassem(insn, ip) + byte_str = (f"{ip:x}").rjust(16) + for k in range(cnt): + byte_str += f" {insn[k]:02x}" + print(f"{byte_str:-40s} {text:-30s}", end=' ') + print(f"{symbol}{offs} ({dso})", end=' ') + else: + print(f"{ip:16x} {symbol}{offs} ({dso})", end=' ') + + addr_correlates_sym = getattr(sample, "addr_correlates_sym", False) + if addr_correlates_sym: + addr = sample.addr + addr_dso = getattr(sample, "addr_dso", "[unknown]") + addr_symbol = getattr(sample, "addr_symbol", "[unknown]") + addr_offs = f"+{sample.addr_symoff:#x}" if hasattr(sample, "addr_symoff") else "" + print(f"=> {addr:x} {addr_symbol}{addr_offs} ({addr_dso}){ipc_str}") + else: + print(ipc_str) + + def print_srccode(self, comm: str, sample: perf.sample_event, + symbol: str, dso: str, with_insn: bool) -> None: + """Print source code info.""" + ip = sample.ip + if symbol == "[unknown]": + start_str = self.common_start_str(comm, sample) + (f"{ip:x}").rjust(16).ljust(40) + else: + offs = f"+{sample.symoff:#x}" if hasattr(sample, "symoff") else "" + start_str = self.common_start_str(comm, sample) + (symbol + offs).ljust(40) + + if with_insn and self.insn and self.disassembler is not None: + try: + insn = sample.insn() + except AttributeError: + insn = None + if insn: + _, text = self.disassem(insn, ip) + start_str += text.ljust(30) + + try: + source_file_name, line_number, source_line = sample.srccode() + except (AttributeError, ValueError): + source_file_name, line_number, source_line = None, 0, None + + if source_file_name: + if self.line_number == line_number and self.source_file_name == source_file_name: + src_str = "" + else: + if len(source_file_name) > 40: + src_file = ("..." + source_file_name[-37:]) + " " + else: + src_file = source_file_name.ljust(41) + if source_line is None: + src_str = src_file + str(line_number).rjust(4) + " <source not found>" + else: + src_str = src_file + str(line_number).rjust(4) + " " + source_line + self.dso = None + elif dso == self.dso: + src_str = "" + else: + src_str = dso + self.dso = dso + + self.line_number = line_number + self.source_file_name = source_file_name + + print(start_str, src_str) + + def do_process_event(self, sample: perf.sample_event) -> None: + """Process event and print info.""" + comm = getattr(sample, "comm", "Unknown") + name = str(sample.evsel) + dso = getattr(sample, "dso", "[unknown]") + symbol = getattr(sample, "symbol", "[unknown]") + + cpu = sample.cpu + if cpu in self.switch_str: + print(self.switch_str[cpu]) + del self.switch_str[cpu] + + try: + raw_buf = sample.raw_buf + except AttributeError: + raw_buf = b"" + + if name.startswith("instructions"): + if self.src: + self.print_srccode(comm, sample, symbol, dso, True) + else: + self.print_instructions_start(comm, sample) + self.print_common_ip(sample, symbol, dso) + elif name.startswith("branches"): + if self.src: + self.print_srccode(comm, sample, symbol, dso, False) + else: + self.print_common_start(comm, sample, name) + self.print_common_ip(sample, symbol, dso) + elif name == "ptwrite": + self.print_common_start(comm, sample, name) + self.print_ptwrite(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "cbr": + self.print_common_start(comm, sample, name) + self.print_cbr(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "mwait": + self.print_common_start(comm, sample, name) + self.print_mwait(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "pwre": + self.print_common_start(comm, sample, name) + self.print_pwre(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "exstop": + self.print_common_start(comm, sample, name) + self.print_exstop(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "pwrx": + self.print_common_start(comm, sample, name) + self.print_pwrx(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "psb": + self.print_common_start(comm, sample, name) + self.print_psb(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "evt": + self.print_common_start(comm, sample, name) + self.print_evt(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "iflag": + self.print_common_start(comm, sample, name) + self.print_iflag(raw_buf) + self.print_common_ip(sample, symbol, dso) + else: + self.print_common_start(comm, sample, name) + self.print_common_ip(sample, symbol, dso) + + def interleave_events(self, sample: perf.sample_event) -> None: + """Interleave output to avoid garbled lines from different CPUs.""" + self.cpu = sample.cpu + ts = sample.time + + if self.time != ts: + self.time = ts + self.flush_stashed_output() + + self.output_pos = 0 + with contextlib.redirect_stdout(io.StringIO()) as self.output: + self.do_process_event(sample) + + self.stash_output() + + def stash_output(self) -> None: + """Stash output for later flushing.""" + output_str = self.output.getvalue()[self.output_pos:] + n = len(output_str) + if n: + self.output_pos += n + if self.cpu not in self.stash_dict: + self.stash_dict[self.cpu] = [] + self.stash_dict[self.cpu].append(output_str) + + def flush_stashed_output(self) -> None: + """Flush stashed output.""" + while self.stash_dict: + cpus = list(self.stash_dict.keys()) + for cpu in cpus: + items = self.stash_dict[cpu] + countdown = self.args.interleave + while len(items) and countdown: + sys.stdout.write(items[0]) + del items[0] + countdown -= 1 + if not items: + del self.stash_dict[cpu] + + def process_event(self, sample: perf.sample_event) -> None: + """Wrapper to handle interleaving and exceptions.""" + try: + if self.args.interleave: + self.interleave_events(sample) + else: + self.do_process_event(sample) + except BrokenPipeError: + # Stop python printing broken pipe errors and traceback + with open(os.devnull, 'w', encoding='utf-8') as f: + sys.stdout = f + sys.exit(1) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--insn-trace", action='store_true') + ap.add_argument("--src-trace", action='store_true') + ap.add_argument("--all-switch-events", action='store_true') + ap.add_argument("--interleave", type=int, nargs='?', const=4, default=0) + args = ap.parse_args() + + analyzer = IntelPTAnalyzer(args) + + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + if args.interleave: + analyzer.flush_stashed_output() + print("End") + except KeyboardInterrupt: + if args.interleave: + analyzer.flush_stashed_output() + print("End") + except Exception as e: + print(f"Error processing events: {e}")
tools/perf/python/libxed.py new file +119 -0
119 lines not on this page · raw message
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 39/58] perf intel-pt-events: Port intel-pt-events/libxed to use python module
Sashiko AI review found 8 potential issue(s): - [High] Accessing fields like `sample.cpu`, `sample.time`, `sample.pid`, and `sample.ip` will immediately crash the script with an `AttributeError`.…
Sashiko AI review found 8 potential issue(s):
- [High] Accessing fields like sample.cpu, sample.time, sample.pid, and sample.ip will immediately crash the script with an AttributeError.
- [High] Event-type conditionals will always evaluate to False due to incorrectly extracting the event name via str(sample.evsel).
- [High] The thread command name (comm) logic is broken and will silently fall back to "Unknown" for all events.
- [High] Instruction bytes containing NUL characters (\x00) will be silently truncated before disassembly.
- [High] Extracting the decoded instruction length relies on a hardcoded internal offset in libxed.py, leading to potential out-of-bounds array reads.
- [High] Missing length checks before struct.unpack_from leads to out-of-bounds reads and script termination on unvalidated payload buffers.
- [Medium] Untrusted payload data is used as a mathematical divisor without being checked for zero.
- [Medium] The exception handler designed to suppress BrokenPipeError traceback output contains a bug that triggers an uncatchable exception during interpreter shutdown. --
commit 5f2caf09eb634efff61968a9fb75db89cffff164
Author: Ian Rogers <[email protected]>
perf intel-pt-events: Port intel-pt-events/libxed to use python module
This commit ports the intel-pt-events and libxed scripts to use a class structure and a standalone python module. It removes Python 2 compatibility and renames methods in libxed.py to snake case.
Quoted 5 lines
tools/perf/python/intel-pt-events.py new file +0
-0
[ ... ]
tools/perf/python/intel-pt-events.py line 79
Quoted 6 lines
tools/perf/python/intel-pt-events.py new file +6
-0+ def print_cbr(self, raw_buf: bytes) -> None: + """Print CBR data.""" + data = struct.unpack_from("<BBBBII", raw_buf) + cbr = data[0] + f = (data[4] + 500) // 1000 + p = ((cbr * 1000 // data[2]) + 5) // 10
Can struct.unpack_from() raise an exception if raw_buf is empty or smaller than the requested format?
Is it possible for data[2] to be zero, triggering a ZeroDivisionError during this calculation?
[ ... ]
Quoted 1 line
+ def
print_evt(self, raw_buf: bytes)-> None:
[ ... ]
Quoted 5 lines
tools/perf/python/intel-pt-events.py new file +5
-0+ data = struct.unpack_from("<BBH", raw_buf) + typ = data[0] & 0x1f + ip_flag = (data[0] & 0x80) >> 7 + vector = data[1] + evd_cnt = data[2]
[ ... ]
tools/perf/python/intel-pt-events.py line 140
Quoted 3 lines
tools/perf/python/intel-pt-events.py new file +3
-0+ pos = 4 + for _ in range(evd_cnt): + data = struct.unpack_from("<QQ", raw_buf, pos)
Are there missing length checks on raw_buf? If evd_cnt is larger than the remaining buffer size, wouldn't struct.unpack_from() raise a struct.error and terminate the session?
[ ... ]
tools/perf/python/intel-pt-events.py line 190
Quoted 6 lines
tools/perf/python/intel-pt-events.py new file +6
-0+ def disassem(self, insn: bytes, ip: int) -> tuple[int, str]: + """Disassemble instruction using LibXED.""" + inst = self.disassembler.instruction() + self.disassembler.set_mode(inst, 0) # Assume 64-bit + buf = create_string_buffer(64) + buf.value = insn
Doesn't assigning insn to buf.value treat it as a null-terminated C string? Since x86 machine code can contain \x00 bytes, won't this prematurely truncate the instruction stream before disassembly? Would assigning to buf.raw be safer?
[ ... ]
tools/perf/python/intel-pt-events.py line 195
Quoted 3 lines
tools/perf/python/intel-pt-events.py new file +3
-0+ def print_common_ip(self, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print IP and symbol info.""" + ip = sample.ip
Do these sample attributes exist on the new perf.sample_event C extension object? Looking at the sample_member_def macro in tools/perf/util/python.c, it seems the properties are exposed with a sample_ prefix, such as sample_ip, sample_cpu, and sample_time. Will accessing sample.ip raise an AttributeError?
[ ... ]
tools/perf/python/intel-pt-events.py line 280
Quoted 4 lines
tools/perf/python/intel-pt-events.py new file +4
-0+ def do_process_event(self, sample: perf.sample_event) -> None: + """Process event and print info.""" + comm = getattr(sample, "comm", "Unknown") + name = str(sample.evsel)
Does the perf.sample_event object expose a comm property? Without the legacy environment manually injecting it into the event dictionary, will this unconditionally fall back to Unknown? Would calling session.process(sample.sample_pid).comm() be required to retrieve the thread name?
Also, doesn't str(sample.evsel) return a formatted string like evsel(instructions) rather than just instructions?
[ ... ]
Quoted 1 line
+ if
name.startswith("instructions"):
If name contains evsel(instructions), will this startswith() check, as well as the exact matches like name == "ptwrite" below, always evaluate to false?
[ ... ]
tools/perf/python/intel-pt-events.py line 396
Quoted 12 lines
tools/perf/python/intel-pt-events.py new file +12
-0+ def process_event(self, sample: perf.sample_event) -> None: + """Wrapper to handle interleaving and exceptions.""" + try: + if self.args.interleave: + self.interleave_events(sample) + else: + self.do_process_event(sample) + except BrokenPipeError: + # Stop python printing broken pipe errors and traceback + with open(os.devnull, 'w', encoding='utf-8') as f: + sys.stdout = f + sys.exit(1)
When the with block ends, f is closed, but sys.stdout still points to it. When sys.exit(1) triggers interpreter shutdown, will it attempt to flush the closed sys.stdout and raise a secondary ValueError, defeating the purpose of suppressing the traceback?
Quoted 5 lines
tools/perf/python/libxed.py new file +0
-0
[ ... ]
Quoted 2 lines
tools/perf/python/libxed.py new file +2
-0+ def disassemble_one(self, inst, bytes_ptr, bytes_cnt, ip): + """Disassemble one instruction."""
[ ... ]
tools/perf/python/libxed.py line 119
Quoted 4 lines
tools/perf/python/libxed.py new file +4
-0+ result = inst.buffer.value.decode('utf-8') + # Return instruction length and the disassembled instruction text + # For now, assume the length is in byte 166 + return inst.xedd[166], result
Is it safe to rely on this hardcoded internal struct offset? If the layout of xed_decoded_inst_t changes across libxed versions and byte 166 contains a garbage value larger than bytes_cnt, will the calling loop in print_common_ip() perform an out-of-bounds read and crash? Should this use the official libxed API function to get the instruction length?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=39
Ian Rogers
reviewerPATCH 40/58 perf net_dropmonitor: Port net_dropmonitor to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/.
- Refactored the script to use a class structure (DropMonitor) to encapsulate state.
- Used perf.session for event processing instead of legacy global handlers.
- Maintained the manual /proc/kallsyms reading and binary search for symbol resolution as in the original script.
- Cleaned up Python 2 compatibility artifacts.
--- tools/perf/python/net_dropmonitor.py | 103 +++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 tools/perf/python/net_dropmonitor.py
tools/perf/python/net_dropmonitor.py new file +103 -0
@@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Monitor the system for dropped packets and produce a report of drop locations and counts. +Ported from tools/perf/scripts/python/net_dropmonitor.py +""" + +import argparse +from collections import defaultdict +from typing import Optional +import perf + + +class DropMonitor: + """Monitors dropped packets and aggregates counts by location.""" + + def __init__(self): + self.drop_log: dict[str, int] = defaultdict(int) + self.kallsyms: list[tuple[int, str]] = [] + + def get_kallsyms_table(self) -> None: + """Read /proc/kallsyms to resolve addresses.""" + try: + with open("/proc/kallsyms", "r", encoding='utf-8') as f: + for line in f: + parts = line.split() + if len(parts) >= 3: + loc = int(parts[0], 16) + name = parts[2] + self.kallsyms.append((loc, name)) + except OSError as e: + print(f"Failed to read /proc/kallsyms: {e}") + return + self.kallsyms.sort() + + def get_sym(self, sloc: str) -> tuple[Optional[str], int]: + """Binary search in kallsyms table.""" + loc = int(sloc) + + # Invariant: kallsyms[i][0] <= loc for all 0 <= i <= start + # kallsyms[i][0] > loc for all end <= i < len(kallsyms) + start, end = -1, len(self.kallsyms) + while end != start + 1: + pivot = (start + end) // 2 + if loc < self.kallsyms[pivot][0]: + end = pivot + else: + start = pivot + + # Now (start == -1 or kallsyms[start][0] <= loc) + # and (start == len(kallsyms) - 1 or loc < kallsyms[start + 1][0]) + if start >= 0: + symloc, name = self.kallsyms[start] + return name, loc - symloc + return None, 0 + + def print_drop_table(self) -> None: + """Print aggregated results.""" + print(f"{'LOCATION':>25} {'OFFSET':>25} {'COUNT':>25}") + for i in sorted(self.drop_log.keys()): + sym, off = self.get_sym(i) + if sym is None: + sym = i + print(f"{sym:>25} {off:>25} {self.drop_log[i]:>25}") + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + if str(sample.evsel) != "evsel(skb:kfree_skb)": + return + + location = getattr(sample, "location", None) + if location is None: + # Try to get it from raw_buf if not exposed directly + # But let's assume it is exposed as in failed-syscalls + return + + slocation = str(location) + self.drop_log[slocation] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Monitor the system for dropped packets and produce a " + "report of drop locations and counts.") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + monitor = DropMonitor() + + try: + print("Starting trace (Ctrl-C to dump results)") + session = perf.session(perf.data(args.input), sample=monitor.process_event) + session.process_events() + + print("Gathering kallsyms data") + monitor.get_kallsyms_table() + monitor.print_drop_table() + except KeyboardInterrupt: + print("\nGathering kallsyms data") + monitor.get_kallsyms_table() + monitor.print_drop_table() + except Exception as e: + print(f"Error processing events: {e}")
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 40/58] perf net_dropmonitor: Port net_dropmonitor to use python module
Sashiko AI review found 2 potential issue(s): - [Medium] The `KeyboardInterrupt` exception handler unintentionally restarts the cleanup phase, leading to duplicated data. - [Medium] Drop locations…
Sashiko AI review found 2 potential issue(s):
- [Medium] The KeyboardInterrupt exception handler unintentionally restarts the cleanup phase, leading to duplicated data.
- [Medium] Drop locations are incorrectly sorted lexicographically as strings instead of numerically.
--
commit 944cd5b52d19260b42c7e0a2807670526ba16f3d
Author: Ian Rogers <[email protected]>
perf net_dropmonitor: Port net_dropmonitor to use python module
This patch ports net_dropmonitor to use the python module by encapsulating state in a DropMonitor class. It replaces legacy global handlers with perf.session for event processing and removes Python 2 compatibility artifacts.
Quoted 5 lines
tools/perf/python/net_dropmonitor.py new file +0
-0
[ ... ]
tools/perf/python/net_dropmonitor.py line 60
Quoted 4 lines
tools/perf/python/net_dropmonitor.py new file +4
-0+ def print_drop_table(self) -> None: + """Print aggregated results.""" + print(f"{'LOCATION':>25} {'OFFSET':>25} {'COUNT':>25}") + for i in sorted(self.drop_log.keys()):
Since drop_log stores string keys, will sorted(self.drop_log.keys()) order the memory locations lexicographically rather than numerically?
[ ... ]
Quoted 1 line
+ def
process_event(self, sample: perf.sample_event)-> None:
[ ... ]
Quoted 2 lines
tools/perf/python/net_dropmonitor.py new file +2
-0+ slocation = str(location) + self.drop_log[slocation] += 1
[ ... ]
tools/perf/python/net_dropmonitor.py line 101
Quoted 12 lines
tools/perf/python/net_dropmonitor.py new file +12
-0+ try: + print("Starting trace (Ctrl-C to dump results)") + session = perf.session(perf.data(args.input), sample=monitor.process_event) + session.process_events() + + print("Gathering kallsyms data") + monitor.get_kallsyms_table() + monitor.print_drop_table() + except KeyboardInterrupt: + print("\nGathering kallsyms data") + monitor.get_kallsyms_table() + monitor.print_drop_table()
If a KeyboardInterrupt occurs while gathering kallsyms data or printing the table, will this exception handler execute the cleanup phase a second time?
Looking at get_kallsyms_table():
def get_kallsyms_table(self) -> None:
...
self.kallsyms.append((loc, name))Because it appends to the kallsyms list, could restarting this phase lead to duplicated entries?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=40
Ian Rogers
reviewerPATCH 41/58 perf netdev-times: Port netdev-times to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/.
- Refactored the script to use a class structure (NetDevTimesAnalyzer) to encapsulate state.
- Used perf.session for event collection and processed them in time order at the end to match legacy behavior.
- Extracted tracepoint fields directly from sample attributes.
- Moved format string constants to module level.
- Cleaned up Python 2 compatibility artifacts (like cmp_to_key).
--- tools/perf/python/netdev-times.py | 469 ++++++++++++++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100755 tools/perf/python/netdev-times.py
tools/perf/python/netdev-times.py new file +469 -0
@@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Display a process of packets and processed time. +It helps us to investigate networking or network device. + +Ported from tools/perf/scripts/python/netdev-times.py +""" + +import argparse +from collections import defaultdict +import sys +import perf + +# Format for displaying rx packet processing +PF_IRQ_ENTRY = " irq_entry(+%.3fmsec irq=%d:%s)" +PF_SOFT_ENTRY = " softirq_entry(+%.3fmsec)" +PF_NAPI_POLL = " napi_poll_exit(+%.3fmsec %s)" +PF_JOINT = " |" +PF_WJOINT = " | |" +PF_NET_RECV = " |---netif_receive_skb(+%.3fmsec skb=%x len=%d)" +PF_NET_RX = " |---netif_rx(+%.3fmsec skb=%x)" +PF_CPY_DGRAM = " | skb_copy_datagram_iovec(+%.3fmsec %d:%s)" +PF_KFREE_SKB = " | kfree_skb(+%.3fmsec location=%x)" +PF_CONS_SKB = " | consume_skb(+%.3fmsec)" + + +class NetDevTimesAnalyzer: + """Analyzes network device events and prints charts.""" + + def __init__(self, cfg: argparse.Namespace): + self.args = cfg + self.show_tx = cfg.tx or (not cfg.tx and not cfg.rx) + self.show_rx = cfg.rx or (not cfg.tx and not cfg.rx) + self.dev = cfg.dev + self.debug = cfg.debug + + self.all_event_list: list[dict] = [] + self.irq_dic: dict[int, list[dict]] = defaultdict(list) + self.net_rx_dic: dict[int, dict] = {} + self.receive_hunk_list: list[dict] = [] + self.rx_skb_list: list[dict] = [] + self.tx_queue_list: list[dict] = [] + self.tx_xmit_list: list[dict] = [] + self.tx_free_list: list[dict] = [] + + self.buffer_budget = 65536 + self.of_count_rx_skb_list = 0 + self.of_count_tx_queue_list = 0 + self.of_count_tx_xmit_list = 0 + + def diff_msec(self, src: int, dst: int) -> float: + """Calculate a time interval(msec) from src(nsec) to dst(nsec).""" + return (dst - src) / 1000000.0 + + def print_transmit(self, hunk: dict) -> None: + """Display a process of transmitting a packet.""" + if self.dev and hunk['dev'].find(self.dev) < 0: + return + queue_t_sec = hunk['queue_t'] // 1000000000 + queue_t_usec = hunk['queue_t'] % 1000000000 // 1000 + print(f"{hunk['dev']:7s} {hunk['len']:5d} " + f"{queue_t_sec:6d}.{queue_t_usec:06d}sec " + f"{self.diff_msec(hunk['queue_t'], hunk['xmit_t']):12.3f}msec " + f"{self.diff_msec(hunk['xmit_t'], hunk['free_t']):12.3f}msec") + + def print_receive(self, hunk: dict) -> None: + """Display a process of received packets and interrupts.""" + show_hunk = False + irq_list = hunk['irq_list'] + if not irq_list: + return + cpu = irq_list[0]['cpu'] + base_t = irq_list[0]['irq_ent_t'] + + if self.dev: + for irq in irq_list: + if irq['name'].find(self.dev) >= 0: + show_hunk = True + break + else: + show_hunk = True + + if not show_hunk: + return + + base_t_sec = base_t // 1000000000 + base_t_usec = base_t % 1000000000 // 1000 + print(f"{base_t_sec}.{base_t_usec:06d}sec cpu={cpu}") + for irq in irq_list: + print(PF_IRQ_ENTRY % + (self.diff_msec(base_t, irq['irq_ent_t']), + irq['irq'], irq['name'])) + print(PF_JOINT) + irq_event_list = irq['event_list'] + for irq_event in irq_event_list: + if irq_event['event'] == 'netif_rx': + print(PF_NET_RX % + (self.diff_msec(base_t, irq_event['time']), + irq_event['skbaddr'])) + print(PF_JOINT) + + print(PF_SOFT_ENTRY % self.diff_msec(base_t, hunk['sirq_ent_t'])) + print(PF_JOINT) + event_list = hunk['event_list'] + for i, event in enumerate(event_list): + if event['event_name'] == 'napi_poll': + print(PF_NAPI_POLL % + (self.diff_msec(base_t, event['event_t']), + event['dev'])) + if i == len(event_list) - 1: + print("") + else: + print(PF_JOINT) + else: + print(PF_NET_RECV % + (self.diff_msec(base_t, event['event_t']), + event['skbaddr'], + event['len'])) + if 'comm' in event: + print(PF_WJOINT) + print(PF_CPY_DGRAM % + (self.diff_msec(base_t, event['comm_t']), + event['pid'], event['comm'])) + elif 'handle' in event: + print(PF_WJOINT) + if event['handle'] == "kfree_skb": + print(PF_KFREE_SKB % + (self.diff_msec(base_t, event['comm_t']), + event['location'])) + elif event['handle'] == "consume_skb": + print(PF_CONS_SKB % + self.diff_msec(base_t, event['comm_t'])) + print(PF_JOINT) + + def handle_irq_handler_entry(self, event: dict) -> None: + """Handle irq:irq_handler_entry event.""" + time = event['time'] + cpu = event['cpu'] + irq = event['irq'] + irq_name = event['irq_name'] + irq_record = {'irq': irq, 'name': irq_name, 'cpu': cpu, + 'irq_ent_t': time, 'event_list': []} + self.irq_dic[cpu].append(irq_record) + + def handle_irq_handler_exit(self, event: dict) -> None: + """Handle irq:irq_handler_exit event.""" + time = event['time'] + cpu = event['cpu'] + irq = event['irq'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + irq_record = self.irq_dic[cpu].pop() + if irq != irq_record['irq']: + return + irq_record['irq_ext_t'] = time + # if an irq doesn't include NET_RX softirq, drop. + if irq_record['event_list']: + self.irq_dic[cpu].append(irq_record) + + def handle_irq_softirq_raise(self, event: dict) -> None: + """Handle irq:softirq_raise event.""" + time = event['time'] + cpu = event['cpu'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + irq_record = self.irq_dic[cpu].pop() + irq_record['event_list'].append({'time': time, 'event': 'sirq_raise'}) + self.irq_dic[cpu].append(irq_record) + + def handle_irq_softirq_entry(self, event: dict) -> None: + """Handle irq:softirq_entry event.""" + time = event['time'] + cpu = event['cpu'] + self.net_rx_dic[cpu] = {'sirq_ent_t': time, 'event_list': []} + + def handle_irq_softirq_exit(self, event: dict) -> None: + """Handle irq:softirq_exit event.""" + time = event['time'] + cpu = event['cpu'] + irq_list = [] + event_list = [] + sirq_ent_t = None + + if cpu in self.irq_dic: + irq_list = self.irq_dic[cpu] + del self.irq_dic[cpu] + if cpu in self.net_rx_dic: + sirq_ent_t = self.net_rx_dic[cpu]['sirq_ent_t'] + event_list = self.net_rx_dic[cpu]['event_list'] + del self.net_rx_dic[cpu] + if not irq_list or not event_list or sirq_ent_t is None: + return + rec_data = {'sirq_ent_t': sirq_ent_t, 'sirq_ext_t': time, + 'irq_list': irq_list, 'event_list': event_list} + self.receive_hunk_list.append(rec_data) + + def handle_napi_poll(self, event: dict) -> None: + """Handle napi:napi_poll event.""" + time = event['time'] + cpu = event['cpu'] + dev_name = event['dev_name'] + work = event['work'] + budget = event['budget'] + if cpu in self.net_rx_dic: + event_list = self.net_rx_dic[cpu]['event_list'] + rec_data = {'event_name': 'napi_poll', + 'dev': dev_name, 'event_t': time, + 'work': work, 'budget': budget} + event_list.append(rec_data) + + def handle_netif_rx(self, event: dict) -> None: + """Handle net:netif_rx event.""" + time = event['time'] + cpu = event['cpu'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + dev_name = event['dev_name'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + irq_record = self.irq_dic[cpu].pop() + irq_record['event_list'].append({'time': time, 'event': 'netif_rx', + 'skbaddr': skbaddr, 'skblen': skblen, + 'dev_name': dev_name}) + self.irq_dic[cpu].append(irq_record) + + def handle_netif_receive_skb(self, event: dict) -> None: + """Handle net:netif_receive_skb event.""" + time = event['time'] + cpu = event['cpu'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + if cpu in self.net_rx_dic: + rec_data = {'event_name': 'netif_receive_skb', + 'event_t': time, 'skbaddr': skbaddr, 'len': skblen} + event_list = self.net_rx_dic[cpu]['event_list'] + event_list.append(rec_data) + self.rx_skb_list.insert(0, rec_data) + if len(self.rx_skb_list) > self.buffer_budget: + self.rx_skb_list.pop() + self.of_count_rx_skb_list += 1 + + def handle_net_dev_queue(self, event: dict) -> None: + """Handle net:net_dev_queue event.""" + time = event['time'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + dev_name = event['dev_name'] + skb = {'dev': dev_name, 'skbaddr': skbaddr, 'len': skblen, 'queue_t': time} + self.tx_queue_list.insert(0, skb) + if len(self.tx_queue_list) > self.buffer_budget: + self.tx_queue_list.pop() + self.of_count_tx_queue_list += 1 + + def handle_net_dev_xmit(self, event: dict) -> None: + """Handle net:net_dev_xmit event.""" + time = event['time'] + skbaddr = event['skbaddr'] + rc = event['rc'] + if rc == 0: # NETDEV_TX_OK + for i, skb in enumerate(self.tx_queue_list): + if skb['skbaddr'] == skbaddr: + skb['xmit_t'] = time + self.tx_xmit_list.insert(0, skb) + del self.tx_queue_list[i] + if len(self.tx_xmit_list) > self.buffer_budget: + self.tx_xmit_list.pop() + self.of_count_tx_xmit_list += 1 + return + + def handle_kfree_skb(self, event: dict) -> None: + """Handle skb:kfree_skb event.""" + time = event['time'] + skbaddr = event['skbaddr'] + comm = event['comm'] + pid = event['pid'] + location = event['location'] + for i, skb in enumerate(self.tx_queue_list): + if skb['skbaddr'] == skbaddr: + del self.tx_queue_list[i] + return + for i, skb in enumerate(self.tx_xmit_list): + if skb['skbaddr'] == skbaddr: + skb['free_t'] = time + self.tx_free_list.append(skb) + del self.tx_xmit_list[i] + return + for i, rec_data in enumerate(self.rx_skb_list): + if rec_data['skbaddr'] == skbaddr: + rec_data.update({'handle': "kfree_skb", + 'comm': comm, 'pid': pid, 'comm_t': time, 'location': location}) + del self.rx_skb_list[i] + return + + def handle_consume_skb(self, event: dict) -> None: + """Handle skb:consume_skb event.""" + time = event['time'] + skbaddr = event['skbaddr'] + for i, skb in enumerate(self.tx_xmit_list): + if skb['skbaddr'] == skbaddr: + skb['free_t'] = time + self.tx_free_list.append(skb) + del self.tx_xmit_list[i] + return + + def handle_skb_copy_datagram_iovec(self, event: dict) -> None: + """Handle skb:skb_copy_datagram_iovec event.""" + time = event['time'] + skbaddr = event['skbaddr'] + comm = event['comm'] + pid = event['pid'] + for i, rec_data in enumerate(self.rx_skb_list): + if skbaddr == rec_data['skbaddr']: + rec_data.update({'handle': "skb_copy_datagram_iovec", + 'comm': comm, 'pid': pid, 'comm_t': time}) + del self.rx_skb_list[i] + return + + def process_events(self) -> None: + """Process all collected events.""" + # order all events in time + self.all_event_list.sort(key=lambda a: a['time']) + + # process all events + for event in self.all_event_list: + name = event['name'] + if name == 'irq:softirq_exit': + self.handle_irq_softirq_exit(event) + elif name == 'irq:softirq_entry': + self.handle_irq_softirq_entry(event) + elif name == 'irq:softirq_raise': + self.handle_irq_softirq_raise(event) + elif name == 'irq:irq_handler_entry': + self.handle_irq_handler_entry(event) + elif name == 'irq:irq_handler_exit': + self.handle_irq_handler_exit(event) + elif name == 'napi:napi_poll': + self.handle_napi_poll(event) + elif name == 'net:netif_receive_skb': + self.handle_netif_receive_skb(event) + elif name == 'net:netif_rx': + self.handle_netif_rx(event) + elif name == 'skb:skb_copy_datagram_iovec': + self.handle_skb_copy_datagram_iovec(event) + elif name == 'net:net_dev_queue': + self.handle_net_dev_queue(event) + elif name == 'net:net_dev_xmit': + self.handle_net_dev_xmit(event) + elif name == 'skb:kfree_skb': + self.handle_kfree_skb(event) + elif name == 'skb:consume_skb': + self.handle_consume_skb(event) + + def print_summary(self) -> None: + """Print charts.""" + self.process_events() + + # display receive hunks + if self.show_rx: + for hunk in self.receive_hunk_list: + self.print_receive(hunk) + + # display transmit hunks + if self.show_tx: + print(" dev len Qdisc " + " netdevice free") + for hunk in self.tx_free_list: + self.print_transmit(hunk) + + if self.debug: + print("debug buffer status") + print("----------------------------") + print(f"xmit Qdisc:remain:{len(self.tx_queue_list)} " + f"overflow:{self.of_count_tx_queue_list}") + print(f"xmit netdevice:remain:{len(self.tx_xmit_list)} " + f"overflow:{self.of_count_tx_xmit_list}") + print(f"receive:remain:{len(self.rx_skb_list)} " + f"overflow:{self.of_count_rx_skb_list}") + + def collect_event(self, sample: perf.sample_event) -> None: + """Collect events into all_event_list.""" + name = str(sample.evsel) + event_data = { + 'name': name[6:-1] if name.startswith("evsel(") else name, + 'time': sample.sample_time, + 'cpu': sample.sample_cpu, + 'pid': sample.sample_pid, + 'comm': getattr(sample, "comm", "Unknown"), + } + + # Extract specific fields based on event type + if name.startswith("evsel(irq:softirq_"): + event_data['vec'] = getattr(sample, "vec", 0) + # Filter for NET_RX + try: + # type: ignore + if perf.symbol_str("irq:softirq_entry", "vec", event_data['vec']) != "NET_RX": + return + except AttributeError: + # Fallback if symbol_str not available or fails + if event_data['vec'] != 3: # NET_RX_SOFTIRQ is usually 3 + return + elif name == "evsel(irq:irq_handler_entry)": + event_data['irq'] = getattr(sample, "irq", -1) + event_data['irq_name'] = getattr(sample, "name", "[unknown]") + elif name == "evsel(irq:irq_handler_exit)": + event_data['irq'] = getattr(sample, "irq", -1) + event_data['ret'] = getattr(sample, "ret", 0) + elif name == "evsel(napi:napi_poll)": + event_data['napi'] = getattr(sample, "napi", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]") + event_data['work'] = getattr(sample, "work", 0) + event_data['budget'] = getattr(sample, "budget", 0) + elif name in ("evsel(net:netif_receive_skb)", "evsel(net:netif_rx)", + "evsel(net:net_dev_queue)"): + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "skblen", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]") + elif name == "evsel(net:net_dev_xmit)": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "skblen", 0) + event_data['rc'] = getattr(sample, "rc", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]") + elif name == "evsel(skb:kfree_skb)": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['location'] = getattr(sample, "location", 0) + event_data['protocol'] = getattr(sample, "protocol", 0) + event_data['reason'] = getattr(sample, "reason", 0) + elif name == "evsel(skb:consume_skb)": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['location'] = getattr(sample, "location", 0) + elif name == "evsel(skb:skb_copy_datagram_iovec)": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "skblen", 0) + + self.all_event_list.append(event_data) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Display a process of packets and processed time.") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("tx", nargs="?", help="show only tx chart") + ap.add_argument("rx", nargs="?", help="show only rx chart") + ap.add_argument("dev", nargs="?", help="show only specified device") + ap.add_argument("debug", nargs="?", help="work with debug mode. It shows buffer status.") + args = ap.parse_args() + + parsed_args = argparse.Namespace(tx=False, rx=False, dev=None, debug=False, input=args.input) + + for arg in sys.argv[1:]: + if arg == 'tx': + parsed_args.tx = True + elif arg == 'rx': + parsed_args.rx = True + elif arg.startswith('dev='): + parsed_args.dev = arg[4:] + elif arg == 'debug': + parsed_args.debug = True + + analyzer = NetDevTimesAnalyzer(parsed_args) + + try: + session = perf.session(perf.data(parsed_args.input), sample=analyzer.collect_event) + session.process_events() + analyzer.print_summary() + except KeyboardInterrupt: + analyzer.print_summary() + except Exception as e: + print(f"Error processing events: {e}")
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 41/58] perf netdev-times: Port netdev-times to use python module
Sashiko AI review found 1 potential issue(s): - [Medium] Tracepoint field mismatch causes all packets to report 0 length and '[unknown]' device, breaking script functionality. --
Sashiko AI review found 1 potential issue(s):
- [Medium] Tracepoint field mismatch causes all packets to report 0 length and '[unknown]' device, breaking script functionality. --
commit 650861cea8b019ec7d7a76a67b9c4572675db6f4
Author: Ian Rogers <[email protected]>
perf netdev-times: Port netdev-times to use python module
This commit ports the netdev-times script to use the perf Python module, encapsulating the state in a class and processing events in time order.
Quoted 5 lines
tools/perf/python/netdev-times.py new file +0
-0
[ ... ]
Quoted 1 line
+ def
collect_event(self, sample: perf.sample_event)-> None:
[ ... ]
tools/perf/python/netdev-times.py line 418
Quoted 5 lines
tools/perf/python/netdev-times.py new file +5
-0+ elif name in ("evsel(net:netif_receive_skb)", "evsel(net:netif_rx)", + "evsel(net:net_dev_queue)"): + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "skblen", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]")
Are the field names skblen and dev_name correct for these tracepoints?
The core networking tracepoints (netif_receive_skb, netif_rx, net_dev_queue) typically expose these fields as len and name.
If getattr falls back to the default values, does this cause all packets to report 0 length and '[unknown]' device, breaking the device filtering logic used later in the script?
tools/perf/python/netdev-times.py line 423
Quoted 5 lines
tools/perf/python/netdev-times.py new file +5
-0+ elif name == "evsel(net:net_dev_xmit)": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "skblen", 0) + event_data['rc'] = getattr(sample, "rc", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]")
Could this block be facing the same tracepoint field mismatch with skblen and dev_name?
[ ... ]
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=41
Ian Rogers
reviewerPATCH 42/58 perf powerpc-hcalls: Port powerpc-hcalls to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/.
- Refactored the script to use a class structure (HCallAnalyzer) to encapsulate state.
- Used perf.session for event processing.
- Tracked hcall entry and exit to calculate duration and aggregate statistics.
- Moved the large hcall_table to a module-level constant HCALL_TABLE.
- Cleaned up Python 2 compatibility artifacts.
--- tools/perf/python/powerpc-hcalls.py | 209 ++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100755 tools/perf/python/powerpc-hcalls.py
tools/perf/python/powerpc-hcalls.py new file +209 -0
@@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +""" +Hypervisor call statistics + +Copyright (C) 2018 Ravi Bangoria, IBM Corporation +Ported from tools/perf/scripts/python/powerpc-hcalls.py +""" + +import argparse +from collections import defaultdict +import perf + +# Hypervisor call table +HCALL_TABLE = { + 4: 'H_REMOVE', + 8: 'H_ENTER', + 12: 'H_READ', + 16: 'H_CLEAR_MOD', + 20: 'H_CLEAR_REF', + 24: 'H_PROTECT', + 28: 'H_GET_TCE', + 32: 'H_PUT_TCE', + 36: 'H_SET_SPRG0', + 40: 'H_SET_DABR', + 44: 'H_PAGE_INIT', + 48: 'H_SET_ASR', + 52: 'H_ASR_ON', + 56: 'H_ASR_OFF', + 60: 'H_LOGICAL_CI_LOAD', + 64: 'H_LOGICAL_CI_STORE', + 68: 'H_LOGICAL_CACHE_LOAD', + 72: 'H_LOGICAL_CACHE_STORE', + 76: 'H_LOGICAL_ICBI', + 80: 'H_LOGICAL_DCBF', + 84: 'H_GET_TERM_CHAR', + 88: 'H_PUT_TERM_CHAR', + 92: 'H_REAL_TO_LOGICAL', + 96: 'H_HYPERVISOR_DATA', + 100: 'H_EOI', + 104: 'H_CPPR', + 108: 'H_IPI', + 112: 'H_IPOLL', + 116: 'H_XIRR', + 120: 'H_MIGRATE_DMA', + 124: 'H_PERFMON', + 220: 'H_REGISTER_VPA', + 224: 'H_CEDE', + 228: 'H_CONFER', + 232: 'H_PROD', + 236: 'H_GET_PPP', + 240: 'H_SET_PPP', + 244: 'H_PURR', + 248: 'H_PIC', + 252: 'H_REG_CRQ', + 256: 'H_FREE_CRQ', + 260: 'H_VIO_SIGNAL', + 264: 'H_SEND_CRQ', + 272: 'H_COPY_RDMA', + 276: 'H_REGISTER_LOGICAL_LAN', + 280: 'H_FREE_LOGICAL_LAN', + 284: 'H_ADD_LOGICAL_LAN_BUFFER', + 288: 'H_SEND_LOGICAL_LAN', + 292: 'H_BULK_REMOVE', + 304: 'H_MULTICAST_CTRL', + 308: 'H_SET_XDABR', + 312: 'H_STUFF_TCE', + 316: 'H_PUT_TCE_INDIRECT', + 332: 'H_CHANGE_LOGICAL_LAN_MAC', + 336: 'H_VTERM_PARTNER_INFO', + 340: 'H_REGISTER_VTERM', + 344: 'H_FREE_VTERM', + 348: 'H_RESET_EVENTS', + 352: 'H_ALLOC_RESOURCE', + 356: 'H_FREE_RESOURCE', + 360: 'H_MODIFY_QP', + 364: 'H_QUERY_QP', + 368: 'H_REREGISTER_PMR', + 372: 'H_REGISTER_SMR', + 376: 'H_QUERY_MR', + 380: 'H_QUERY_MW', + 384: 'H_QUERY_HCA', + 388: 'H_QUERY_PORT', + 392: 'H_MODIFY_PORT', + 396: 'H_DEFINE_AQP1', + 400: 'H_GET_TRACE_BUFFER', + 404: 'H_DEFINE_AQP0', + 408: 'H_RESIZE_MR', + 412: 'H_ATTACH_MCQP', + 416: 'H_DETACH_MCQP', + 420: 'H_CREATE_RPT', + 424: 'H_REMOVE_RPT', + 428: 'H_REGISTER_RPAGES', + 432: 'H_DISABLE_AND_GETC', + 436: 'H_ERROR_DATA', + 440: 'H_GET_HCA_INFO', + 444: 'H_GET_PERF_COUNT', + 448: 'H_MANAGE_TRACE', + 468: 'H_FREE_LOGICAL_LAN_BUFFER', + 472: 'H_POLL_PENDING', + 484: 'H_QUERY_INT_STATE', + 580: 'H_ILLAN_ATTRIBUTES', + 592: 'H_MODIFY_HEA_QP', + 596: 'H_QUERY_HEA_QP', + 600: 'H_QUERY_HEA', + 604: 'H_QUERY_HEA_PORT', + 608: 'H_MODIFY_HEA_PORT', + 612: 'H_REG_BCMC', + 616: 'H_DEREG_BCMC', + 620: 'H_REGISTER_HEA_RPAGES', + 624: 'H_DISABLE_AND_GET_HEA', + 628: 'H_GET_HEA_INFO', + 632: 'H_ALLOC_HEA_RESOURCE', + 644: 'H_ADD_CONN', + 648: 'H_DEL_CONN', + 664: 'H_JOIN', + 676: 'H_VASI_STATE', + 688: 'H_ENABLE_CRQ', + 696: 'H_GET_EM_PARMS', + 720: 'H_SET_MPP', + 724: 'H_GET_MPP', + 748: 'H_HOME_NODE_ASSOCIATIVITY', + 756: 'H_BEST_ENERGY', + 764: 'H_XIRR_X', + 768: 'H_RANDOM', + 772: 'H_COP', + 788: 'H_GET_MPP_X', + 796: 'H_SET_MODE', + 61440: 'H_RTAS', +} + + +class HCallAnalyzer: + """Analyzes hypervisor calls and aggregates statistics.""" + + def __init__(self): + # output: {opcode: {'min': min, 'max': max, 'time': time, 'cnt': cnt}} + self.output = defaultdict(lambda: {'time': 0, 'cnt': 0, 'min': float('inf'), 'max': 0}) + # d_enter: {cpu: {opcode: nsec}} + self.d_enter = defaultdict(dict) + self.print_ptrn = '%-28s%10s%10s%10s%10s' + + def hcall_table_lookup(self, opcode: int) -> str: + """Lookup hcall name by opcode.""" + return HCALL_TABLE.get(opcode, str(opcode)) + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + name = str(sample.evsel) + cpu = sample.sample_cpu + time = sample.sample_time + opcode = getattr(sample, "opcode", -1) + + if opcode == -1: + return + + if name == "evsel(powerpc:hcall_entry)": + self.d_enter[cpu][opcode] = time + elif name == "evsel(powerpc:hcall_exit)": + if cpu in self.d_enter and opcode in self.d_enter[cpu]: + diff = time - self.d_enter[cpu][opcode] + del self.d_enter[cpu][opcode] + + stats = self.output[opcode] + stats['time'] += diff + stats['cnt'] += 1 + if diff < stats['min']: + stats['min'] = diff + if diff > stats['max']: + stats['max'] = diff + + def print_summary(self) -> None: + """Print aggregated statistics.""" + print(self.print_ptrn % ('hcall', 'count', 'min(ns)', 'max(ns)', 'avg(ns)')) + print('-' * 68) + for opcode in sorted(self.output.keys()): + h_name = self.hcall_table_lookup(opcode) + stats = self.output[opcode] + time = stats['time'] + cnt = stats['cnt'] + min_t = stats['min'] + max_t = stats['max'] + + # Avoid float representation for large integers if possible, + # or use formatted strings. Legacy used time//cnt. + avg_t = time // cnt if cnt > 0 else 0 + + # If min was not updated, it remains inf, but cnt should be > 0 if in output + if min_t == float('inf'): + min_t = 0 + + print(self.print_ptrn % (h_name, cnt, int(min_t), int(max_t), avg_t)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Hypervisor call statistics") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + analyzer = HCallAnalyzer() + + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + analyzer.print_summary() + except KeyboardInterrupt: + analyzer.print_summary() + except Exception as e: + print(f"Error processing events: {e}")
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
Ian Rogers
reviewerPATCH 43/58 perf sched-migration: Port sched-migration/SchedGui to use python module
In reply to Ian RogersPorted from tools/perf/scripts/python/ and its Util lib.
- Refactored sched-migration.py to use a class structure (SchedMigrationAnalyzer) to encapsulate state.
- Used perf.session for event processing.
- Ported SchedGui.py to the same directory to keep it as a local dependency.
- Made wxPython dependency optional in sched-migration.py, printing a message if it's missing instead of failing with ImportError.
- Cleaned up Python 2 compatibility artifacts.
--- tools/perf/python/SchedGui.py | 180 +++++++++++ tools/perf/python/sched-migration.py | 466 +++++++++++++++++++++++++++ 2 files changed, 646 insertions(+) create mode 100755 tools/perf/python/SchedGui.py create mode 100755 tools/perf/python/sched-migration.py
tools/perf/python/SchedGui.py new file +180 -0
@@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# SchedGui.py - Python extension for perf script, basic GUI code for +# traces drawing and overview. +# +# Copyright (C) 2010 by Frederic Weisbecker <[email protected]> +# +# Ported to modern directory structure. + +try: + import wx +except ImportError: + raise ImportError("You need to install the wxpython lib for this script") + + +class RootFrame(wx.Frame): + Y_OFFSET = 100 + RECT_HEIGHT = 100 + RECT_SPACE = 50 + EVENT_MARKING_WIDTH = 5 + + def __init__(self, sched_tracer, title, parent=None, id=-1): + wx.Frame.__init__(self, parent, id, title) + + (self.screen_width, self.screen_height) = wx.GetDisplaySize() + self.screen_width -= 10 + self.screen_height -= 10 + self.zoom = 0.5 + self.scroll_scale = 20 + self.sched_tracer = sched_tracer + self.sched_tracer.set_root_win(self) + (self.ts_start, self.ts_end) = sched_tracer.interval() + self.update_width_virtual() + self.nr_rects = sched_tracer.nr_rectangles() + 1 + self.height_virtual = RootFrame.Y_OFFSET + (self.nr_rects * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)) + + # whole window panel + self.panel = wx.Panel(self, size=(self.screen_width, self.screen_height)) + + # scrollable container + self.scroll = wx.ScrolledWindow(self.panel) + self.scroll.SetScrollbars(self.scroll_scale, self.scroll_scale, self.width_virtual / self.scroll_scale, self.height_virtual / self.scroll_scale) + self.scroll.EnableScrolling(True, True) + self.scroll.SetFocus() + + # scrollable drawing area + self.scroll_panel = wx.Panel(self.scroll, size=(self.screen_width - 15, self.screen_height / 2)) + self.scroll_panel.Bind(wx.EVT_PAINT, self.on_paint) + self.scroll_panel.Bind(wx.EVT_KEY_DOWN, self.on_key_press) + self.scroll_panel.Bind(wx.EVT_LEFT_DOWN, self.on_mouse_down) + self.scroll.Bind(wx.EVT_PAINT, self.on_paint) + self.scroll.Bind(wx.EVT_KEY_DOWN, self.on_key_press) + self.scroll.Bind(wx.EVT_LEFT_DOWN, self.on_mouse_down) + + self.scroll.Fit() + self.Fit() + + self.scroll_panel.SetDimensions(-1, -1, self.width_virtual, self.height_virtual, wx.SIZE_USE_EXISTING) + + self.txt = None + + self.Show(True) + + def us_to_px(self, val): + return val / (10 ** 3) * self.zoom + + def px_to_us(self, val): + return (val / self.zoom) * (10 ** 3) + + def scroll_start(self): + (x, y) = self.scroll.GetViewStart() + return (x * self.scroll_scale, y * self.scroll_scale) + + def scroll_start_us(self): + (x, y) = self.scroll_start() + return self.px_to_us(x) + + def paint_rectangle_zone(self, nr, color, top_color, start, end): + offset_px = self.us_to_px(start - self.ts_start) + width_px = self.us_to_px(end - self.ts_start) + + offset_py = RootFrame.Y_OFFSET + (nr * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)) + width_py = RootFrame.RECT_HEIGHT + + dc = self.dc + + if top_color is not None: + (r, g, b) = top_color + top_color = wx.Colour(r, g, b) + brush = wx.Brush(top_color, wx.SOLID) + dc.SetBrush(brush) + dc.DrawRectangle(offset_px, offset_py, width_px, RootFrame.EVENT_MARKING_WIDTH) + width_py -= RootFrame.EVENT_MARKING_WIDTH + offset_py += RootFrame.EVENT_MARKING_WIDTH + + (r, g, b) = color + color = wx.Colour(r, g, b) + brush = wx.Brush(color, wx.SOLID) + dc.SetBrush(brush) + dc.DrawRectangle(offset_px, offset_py, width_px, width_py) + + def update_rectangles(self, dc, start, end): + start += self.ts_start + end += self.ts_start + self.sched_tracer.fill_zone(start, end) + + def on_paint(self, event): + dc = wx.PaintDC(self.scroll_panel) + self.dc = dc + + width = min(self.width_virtual, self.screen_width) + (x, y) = self.scroll_start() + start = self.px_to_us(x) + end = self.px_to_us(x + width) + self.update_rectangles(dc, start, end) + + def rect_from_ypixel(self, y): + y -= RootFrame.Y_OFFSET + rect = y / (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE) + height = y % (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE) + + if rect < 0 or rect > self.nr_rects - 1 or height > RootFrame.RECT_HEIGHT: + return -1 + + return rect + + def update_summary(self, txt): + if self.txt: + self.txt.Destroy() + self.txt = wx.StaticText(self.panel, -1, txt, (0, (self.screen_height / 2) + 50)) + + def on_mouse_down(self, event): + (x, y) = event.GetPositionTuple() + rect = self.rect_from_ypixel(y) + if rect == -1: + return + + t = self.px_to_us(x) + self.ts_start + + self.sched_tracer.mouse_down(rect, t) + + def update_width_virtual(self): + self.width_virtual = self.us_to_px(self.ts_end - self.ts_start) + + def __zoom(self, x): + self.update_width_virtual() + (xpos, ypos) = self.scroll.GetViewStart() + xpos = self.us_to_px(x) / self.scroll_scale + self.scroll.SetScrollbars(self.scroll_scale, self.scroll_scale, self.width_virtual / self.scroll_scale, self.height_virtual / self.scroll_scale, xpos, ypos) + self.Refresh() + + def zoom_in(self): + x = self.scroll_start_us() + self.zoom *= 2 + self.__zoom(x) + + def zoom_out(self): + x = self.scroll_start_us() + self.zoom /= 2 + self.__zoom(x) + + def on_key_press(self, event): + key = event.GetRawKeyCode() + if key == ord("+"): + self.zoom_in() + return + if key == ord("-"): + self.zoom_out() + return + + key = event.GetKeyCode() + (x, y) = self.scroll.GetViewStart() + if key == wx.WXK_RIGHT: + self.scroll.Scroll(x + 1, y) + elif key == wx.WXK_LEFT: + self.scroll.Scroll(x - 1, y) + elif key == wx.WXK_DOWN: + self.scroll.Scroll(x, y + 1) + elif key == wx.WXK_UP: + self.scroll.Scroll(x, y - 1)
tools/perf/python/sched-migration.py new file +466 -0
466 lines not on this page · raw message
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 43/58] perf sched-migration: Port sched-migration/SchedGui to use python module
Sashiko AI review found 5 potential issue(s): - [High] Python 3 float division causes TypeErrors in wxPython UI layout and generates invalid fractional CPU keys, breaking click functionality.…
Sashiko AI review found 5 potential issue(s):
- [High] Python 3 float division causes TypeErrors in wxPython UI layout and generates invalid fractional CPU keys, breaking click functionality.
- [High] The application crashes with an IndexError on startup if perf.data lacks scheduler events.
- [High] Event rectangle widths are calculated incorrectly, causing them to stretch and overlap across the timeline.
- [High] __migrate_in incorrectly mutates a shared RunqueueSnapshot instance, retroactively corrupting the event history of previous timeslices.
- [High] Trace rendering completely disappears if the view's start time is requested slightly before the first recorded event. --
commit da3c00ae67ea12adb65dbfb69aac20ceea3931bb
Author: Ian Rogers <[email protected]>
perf sched-migration: Port sched-migration/SchedGui to use python module
This commit ports sched-migration and SchedGui to use the perf.session Python module structure. It refactors the tool to use class encapsulation and cleans up Python 2 compatibility artifacts.
Quoted 5 lines
tools/perf/python/SchedGui.py new file +0
-0
[ ... ]
tools/perf/python/SchedGui.py line 47
Quoted 8 lines
tools/perf/python/SchedGui.py new file +8
-0+ # scrollable container + self.scroll = wx.ScrolledWindow(self.panel) + self.scroll.SetScrollbars(self.scroll_scale, self.scroll_scale, self.width_virtual / self.scroll_scale, self.height_virtual / self.scroll_scale) + self.scroll.EnableScrolling(True, True) + self.scroll.SetFocus() + + # scrollable drawing area + self.scroll_panel = wx.Panel(self.scroll, size=(self.screen_width - 15, self.screen_height / 2))
In Python 3, division (/) returns a float. Does wxPython Phoenix strict type checking cause a TypeError when a float is passed for the height here? Should this use integer division (//) instead?
[ ... ]
tools/perf/python/SchedGui.py line 80
Quoted 3 lines
tools/perf/python/SchedGui.py new file +3
-0+ def paint_rectangle_zone(self, nr, color, top_color, start, end): + offset_px = self.us_to_px(start - self.ts_start) + width_px = self.us_to_px(end - self.ts_start)
Does this calculate the absolute distance from the start of the trace to the event's end time, rather than its duration?
If so, will this cause the drawn rectangles to be excessively wide and overlap across the timeline? Should width_px be calculated as self.us_to_px(end - start)?
Quoted 2 lines
tools/perf/python/SchedGui.py new file +2
-0+ + offset_py = RootFrame.Y_OFFSET + (nr * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE))
[ ... ]
tools/perf/python/SchedGui.py line 119
Quoted 3 lines
tools/perf/python/SchedGui.py new file +3
-0+ def rect_from_ypixel(self, y): + y -= RootFrame.Y_OFFSET + rect = y / (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)
Since this uses float division (/), will it return a fractional value?
When this is used in mouse_down() to index into ts.rqs[cpu], will it fail to find the integer CPU key, resulting in an empty runqueue snapshot on click? Would integer division (//) be more appropriate here?
Quoted 1 line
+ height = y % (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)
[ ... ]
Quoted 5 lines
tools/perf/python/sched-migration.py new file +0
-0
[ ... ]
tools/perf/python/sched-migration.py line 168
Quoted 4 lines
tools/perf/python/sched-migration.py new file +4
-0+ def __migrate_in(self, new: int, event): + if new in self.tasks: + self.event = event + return self
Since TimeSlice instances share RunqueueSnapshot objects to save memory, does mutating self.event here corrupt the recorded event history for all previous timeslices that reference this same snapshot?
Should this return a newly instantiated RunqueueSnapshot instead?
Quoted 3 lines
tools/perf/python/sched-migration.py new file +3
-0+ next_tasks = self.tasks + tuple([new]) + + return RunqueueSnapshot(next_tasks, event)
[ ... ]
tools/perf/python/sched-migration.py line 293
Quoted 12 lines
tools/perf/python/sched-migration.py new file +12
-0+ def find_time_slice(self, ts: int) -> int: + """Binary search for time slice containing timestamp.""" + start = 0 + end = len(self.data) + found = -1 + searching = True + while searching: + if start in (end, end - 1): + searching = False + + i = (end + start) // 2 + if self.data[i].start <= ts <= self.data[i].end:
If the parsed perf.data file lacks scheduler events, self.data is empty. Will this loop fail to break out when searching is set to False, leading to an IndexError when attempting to access self.data[i] here?
Quoted 2 lines
tools/perf/python/sched-migration.py new file +2
-0+ found = i + break
[ ... ]
tools/perf/python/sched-migration.py line 351
Quoted 5 lines
tools/perf/python/sched-migration.py new file +5
-0+ def fill_zone(self, start: int, end: int): + """Fill zone in GUI.""" + i = self.find_time_slice(start) + if i == -1: + return
If the user scrolls such that the requested start time is slightly before the first recorded event, find_time_slice() returns -1. Does returning early here cause the timeline to abruptly go completely blank, even if the end timestamp still overlaps with valid data?
Should it fall back to starting at index 0 instead of returning?
Quoted 2 lines
tools/perf/python/sched-migration.py new file +2
-0+ + for idx in range(i, len(self.data)):
[ ... ]
tools/perf/python/sched-migration.py line 369
Quoted 3 lines
tools/perf/python/sched-migration.py new file +3
-0+ def nr_rectangles(self) -> int: + """Return maximum CPU number.""" + last_ts = self.data[-1]
Similarly, if self.data is empty because no events were parsed, will accessing self.data[-1] raise an IndexError and crash the application on startup?
Quoted 4 lines
tools/perf/python/sched-migration.py new file +4
-0+ max_cpu = 0 + for cpu in last_ts.rqs: + max_cpu = max(max_cpu, cpu) + return max_cpu
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=43
Ian Rogers
reviewerPATCH 44/58 perf sctop: Port sctop to use python module
In reply to Ian RogersPort sctop.py from tools/perf/scripts/python/ to tools/perf/python/, refactoring it to use a class-based structure (SCTopAnalyzer) and the perf.session API.
Also add support for live mode using the LiveSession helper when no input file is specified, with a fallback strategy for tracepoint names (raw_syscalls:sys_enter or syscalls:sys_enter) to support different systems.
--- tools/perf/python/sctop.py | 149 +++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100755 tools/perf/python/sctop.py
tools/perf/python/sctop.py new file +149 -0
@@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +System call top + +Periodically displays system-wide system call totals, broken down by +syscall. If a [comm] arg is specified, only syscalls called by +[comm] are displayed. If an [interval] arg is specified, the display +will be refreshed every [interval] seconds. The default interval is +3 seconds. + +Ported from tools/perf/scripts/python/sctop.py +""" + +import argparse +from collections import defaultdict +import sys +import threading +import perf +from perf_live import LiveSession + + + + +class SCTopAnalyzer: + """Periodically displays system-wide system call totals.""" + + def __init__(self, for_comm: str | None, interval: int): + self.for_comm = for_comm + self.interval = interval + self.syscalls: dict[int, int] = defaultdict(int) + self.lock = threading.Lock() + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.print_syscall_totals) + + def syscall_name(self, syscall_id: int) -> str: + """Lookup syscall name by ID.""" + try: + return perf.syscall_name(syscall_id) + except Exception: # pylint: disable=broad-exception-caught + pass + return str(syscall_id) + + def process_event(self, sample: perf.sample_event) -> None: + """Collect syscall events.""" + name = str(sample.evsel) + comm = getattr(sample, "comm", "Unknown") + syscall_id = getattr(sample, "id", -1) + + if syscall_id == -1: + return + + if name in ("evsel(raw_syscalls:sys_enter)", "evsel(syscalls:sys_enter)"): + if self.for_comm is not None and comm != self.for_comm: + return + with self.lock: + self.syscalls[syscall_id] += 1 + + def print_syscall_totals(self): + """Periodically print syscall totals.""" + while not self.stop_event.is_set(): + # Clear terminal + print("\x1b[2J\x1b[H", end="") + + if self.for_comm is not None: + print(f"\nsyscall events for {self.for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'event':40s} {'count':10s}") + print(f"{'-' * 40:40s} {'-' * 10:10s}") + + with self.lock: + current_syscalls = list(self.syscalls.items()) + self.syscalls.clear() + + current_syscalls.sort(key=lambda kv: (kv[1], kv[0]), reverse=True) + + for syscall_id, val in current_syscalls: + print(f"{self.syscall_name(syscall_id):<40s} {val:10d}") + + self.stop_event.wait(self.interval) + + def start(self): + """Start the background thread.""" + self.thread.start() + + def stop(self): + """Stop the background thread.""" + self.stop_event.set() + self.thread.join() + + +def main(): + """Main function.""" + ap = argparse.ArgumentParser(description="System call top") + ap.add_argument("args", nargs="*", help="[comm] [interval] or [interval]") + ap.add_argument("-i", "--input", help="Input file name") + args = ap.parse_args() + + for_comm = None + default_interval = 3 + interval = default_interval + + if len(args.args) > 2: + print("Usage: perf script -s sctop.py [comm] [interval]") + sys.exit(1) + + if len(args.args) > 1: + for_comm = args.args[0] + try: + interval = int(args.args[1]) + except ValueError: + print(f"Invalid interval: {args.args[1]}") + sys.exit(1) + elif len(args.args) > 0: + try: + interval = int(args.args[0]) + except ValueError: + for_comm = args.args[0] + interval = default_interval + + analyzer = SCTopAnalyzer(for_comm, interval) + analyzer.start() + + try: + if args.input: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + else: + try: + live_session = LiveSession( + "raw_syscalls:sys_enter", sample_callback=analyzer.process_event + ) + except OSError: + live_session = LiveSession( + "syscalls:sys_enter", sample_callback=analyzer.process_event + ) + live_session.run() + except KeyboardInterrupt: + pass + except IOError as e: + print(f"Error: {e}") + finally: + analyzer.stop() + + +if __name__ == "__main__": + main()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 44/58] perf sctop: Port sctop to use python module
Sashiko AI review found 1 potential issue(s): - [High] Silent data loss and missing output in offline trace mode due to incorrect background thread synchronization. --
Sashiko AI review found 1 potential issue(s):
- [High] Silent data loss and missing output in offline trace mode due to incorrect background thread synchronization. --
commit 49888999983c8368caac2c5b2d68dc07aa8060d6
Author: Ian Rogers <[email protected]>
perf sctop: Port sctop to use python module
This commit ports sctop.py to tools/perf/python/, refactoring it to use a class-based structure and the perf.session API. It also adds support for live mode using the LiveSession helper when no input file is specified.
Quoted 5 lines
tools/perf/python/sctop.py new file +0
-0
[ ... ]
Quoted 3 lines
tools/perf/python/sctop.py new file +3
-0+ def print_syscall_totals(self): + """Periodically print syscall totals.""" + while not self.stop_event.is_set():
[ ... ]
tools/perf/python/sctop.py line 82
Quoted 4 lines
tools/perf/python/sctop.py new file +4
-0+ for syscall_id, val in current_syscalls: + print(f"{self.syscall_name(syscall_id):<40s} {val:10d}") + + self.stop_event.wait(self.interval)
Does this code drop the final batch of accumulated syscall events when running in offline mode?
When process_events() completes, the script reaches the finally block and calls stop():
tools/perf/python/sctop.py line 91
Quoted 4 lines
tools/perf/python/sctop.py new file +4
-0+ def stop(self): + """Stop the background thread.""" + self.stop_event.set() + self.thread.join()
Setting stop_event causes stop_event.wait(self.interval) to return immediately. The loop condition evaluates to false, and the thread exits without printing any events that were gathered since the last loop iteration.
If the trace file takes less than the default 3 seconds to parse, would the script exit silently without producing any output at all?
[ ... ]
Quoted 4 lines
tools/perf/python/sctop.py new file +4
-0+ try: + if args.input: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events()
[ ... ]
tools/perf/python/sctop.py line 145
Quoted 2 lines
tools/perf/python/sctop.py new file +2
-0+ finally: + analyzer.stop()
Also, since process_events() parses the trace file as fast as the CPU allows during offline mode, does relying on a wall-clock wait() mean the interval sizes are dependent on the host's parsing speed rather than the trace duration?
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=44
Ian Rogers
reviewerPATCH 45/58 perf stackcollapse: Port stackcollapse to use python module
In reply to Ian RogersModernize the legacy stackcollapse.py trace script by refactoring it into a class-based architecture (StackCollapseAnalyzer). The script uses perf.session for event processing and aggregates call stacks to produce output suitable for flame graphs.
--- tools/perf/python/stackcollapse.py | 120 +++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 tools/perf/python/stackcollapse.py
tools/perf/python/stackcollapse.py new file +120 -0
@@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +stackcollapse.py - format perf samples with one line per distinct call stack + +This script's output has two space-separated fields. The first is a semicolon +separated stack including the program name (from the "comm" field) and the +function names from the call stack. The second is a count: + + swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 2 + +The file is sorted according to the first field. + +Ported from tools/perf/scripts/python/stackcollapse.py +""" + +import argparse +from collections import defaultdict +import sys +import perf + + +class StackCollapseAnalyzer: + """Accumulates call stacks and prints them collapsed.""" + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.lines: dict[str, int] = defaultdict(int) + + def tidy_function_name(self, sym: str, dso: str) -> str: + """Beautify function names based on options.""" + if sym is None: + sym = "[unknown]" + + sym = sym.replace(";", ":") + if self.args.tidy_java: + # Beautify Java signatures + sym = sym.replace("<", "") + sym = sym.replace(">", "") + if sym.startswith("L") and "/" in sym: + sym = sym[1:] + try: + sym = sym[:sym.index("(")] + except ValueError: + pass + + if self.args.annotate_kernel and dso == "[kernel.kallsyms]": + return sym + "_[k]" + return sym + + def process_event(self, sample: perf.sample_event) -> None: + """Collect call stack for each sample.""" + stack = [] + if hasattr(sample, "callchain"): + for node in sample.callchain: + stack.append(self.tidy_function_name(node.symbol, node.dso)) + else: + # Fallback if no callchain + sym = getattr(sample, "symbol", "[unknown]") + dso = getattr(sample, "dso", "[unknown]") + stack.append(self.tidy_function_name(sym, dso)) + + if self.args.include_comm: + comm = getattr(sample, "comm", "Unknown").replace(" ", "_") + sep = "-" + if self.args.include_pid: + comm = f"{comm}{sep}{getattr(sample, 'sample_pid', 0)}" + sep = "/" + if self.args.include_tid: + comm = f"{comm}{sep}{getattr(sample, 'sample_tid', 0)}" + stack.append(comm) + + stack_string = ";".join(reversed(stack)) + self.lines[stack_string] += 1 + + def print_totals(self) -> None: + """Print sorted collapsed stacks.""" + for stack in sorted(self.lines): + print(f"{stack} {self.lines[stack]}") + + +def main(): + """Main function.""" + ap = argparse.ArgumentParser( + description="Format perf samples with one line per distinct call stack" + ) + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--include-tid", action="store_true", help="include thread id in stack") + ap.add_argument("--include-pid", action="store_true", help="include process id in stack") + ap.add_argument("--no-comm", dest="include_comm", action="store_false", default=True, + help="do not separate stacks according to comm") + ap.add_argument("--tidy-java", action="store_true", help="beautify Java signatures") + ap.add_argument("--kernel", dest="annotate_kernel", action="store_true", + help="annotate kernel functions with _[k]") + + args = ap.parse_args() + + if args.include_tid and not args.include_comm: + print("requesting tid but not comm is invalid", file=sys.stderr) + sys.exit(1) + if args.include_pid and not args.include_comm: + print("requesting pid but not comm is invalid", file=sys.stderr) + sys.exit(1) + + analyzer = StackCollapseAnalyzer(args) + + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + except IOError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + pass + + analyzer.print_totals() + + +if __name__ == "__main__": + main()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 45/58] perf stackcollapse: Port stackcollapse to use python module
Sashiko AI review found 2 potential issue(s): - [Medium] Script crashes with a TypeError on samples without callchains due to a flawed hasattr() check. - [Medium] All samples are incorrectly grouped…
Sashiko AI review found 2 potential issue(s):
- [Medium] Script crashes with a TypeError on samples without callchains due to a flawed hasattr() check.
- [Medium] All samples are incorrectly grouped under an 'Unknown' process name because perf.sample_event lacks a 'comm' attribute. --
commit 83683025e1c59392ea964951295fc45bfce78ff1
Author: Ian Rogers <[email protected]>
perf stackcollapse: Port stackcollapse to use python module
Modernize the legacy stackcollapse.py trace script by refactoring it into a class-based architecture. The script uses perf.session for event processing and aggregates call stacks to produce output suitable for flame graphs.
Quoted 6 lines
tools/perf/python/stackcollapse.py new file +0
-0@@ -0,0 +1,120 @@
[ ... ]
tools/perf/python/stackcollapse.py line 55
Quoted 5 lines
tools/perf/python/stackcollapse.py new file +5
-0+ def process_event(self, sample: perf.sample_event) -> None: + """Collect call stack for each sample.""" + stack = [] + if hasattr(sample, "callchain"): + for node in sample.callchain:
Does this code crash when a sample lacks a callchain?
The perf.sample_event object exposes callchain as a C API property descriptor. If a sample lacks a callchain, pyrf_sample_event__get_callchain explicitly returns Py_None.
Because returning None does not raise an AttributeError, hasattr(sample, "callchain") evaluates to True. The script then attempts to iterate over None in the for loop, which raises a TypeError and terminates the script.
tools/perf/python/stackcollapse.py line 64
Quoted 9 lines
tools/perf/python/stackcollapse.py new file +9
-0+ stack.append(self.tidy_function_name(node.symbol, node.dso)) + else: + # Fallback if no callchain + sym = getattr(sample, "symbol", "[unknown]") + dso = getattr(sample, "dso", "[unknown]") + stack.append(self.tidy_function_name(sym, dso)) + + if self.args.include_comm: + comm = getattr(sample, "comm", "Unknown").replace(" ", "_")
Are all samples grouped under an "Unknown" process name?
The C extension's perf.sample_event type does not expose a comm attribute, unlike the legacy perf script dictionary. Consequently, getattr will silently and perpetually default to "Unknown", preventing proper process separation in the collapsed output.
Quoted 4 lines
tools/perf/python/stackcollapse.py new file +4
-0+ sep = "-" + if self.args.include_pid: + comm = f"{comm}{sep}{getattr(sample, 'sample_pid', 0)}" + sep = "/"
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=45
Ian Rogers
reviewerPATCH 46/58 perf task-analyzer: Port task-analyzer to use python module
In reply to Ian RogersPorted task-analyzer.py from tools/perf/scripts/python to tools/perf/python. Refactored to class-based architecture. Added support for both file mode (using perf.session) and live mode (using evlist.read_on_cpu). Accesses tracepoint fields directly from sample object.
Update task-analyzer testing to use command rather than script version, this allows the perf.data file not to be in the same directory as the test is run.
--- tools/perf/python/task-analyzer.py | 529 +++++++++++++++++++ tools/perf/tests/shell/test_task_analyzer.sh | 71 +-- 2 files changed, 570 insertions(+), 30 deletions(-) create mode 100755 tools/perf/python/task-analyzer.py
tools/perf/python/task-analyzer.py new file +529 -0
@@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# task-analyzer.py - comprehensive perf tasks analysis +# Copyright (c) 2022, Hagen Paul Pfeifer <[email protected]> +# Licensed under the terms of the GNU GPL License version 2 +# +# Usage: +# +# perf record -e sched:sched_switch -a -- sleep 10 +# perf script report task-analyzer +# +"""Comprehensive perf tasks analysis.""" + +import argparse +from contextlib import contextmanager +import decimal +import os +import string +import sys +from typing import Any, Optional +import perf + + +# Columns will have a static size to align everything properly +# Support of 116 days of active update with nano precision +LEN_SWITCHED_IN = len("9999999.999999999") +LEN_SWITCHED_OUT = len("9999999.999999999") +LEN_CPU = len("000") +LEN_PID = len("maxvalue") +LEN_TID = len("maxvalue") +LEN_COMM = len("max-comms-length") +LEN_RUNTIME = len("999999.999") +# Support of 3.45 hours of timespans +LEN_OUT_IN = len("99999999999.999") +LEN_OUT_OUT = len("99999999999.999") +LEN_IN_IN = len("99999999999.999") +LEN_IN_OUT = len("99999999999.999") + +class Timespans: + """Tracks elapsed time between occurrences of the same task.""" + def __init__(self, args: argparse.Namespace, time_unit: str) -> None: + self.args = args + self.time_unit = time_unit + self._last_start: Optional[decimal.Decimal] = None + self._last_finish: Optional[decimal.Decimal] = None + self.current = { + 'out_out': decimal.Decimal(-1), + 'in_out': decimal.Decimal(-1), + 'out_in': decimal.Decimal(-1), + 'in_in': decimal.Decimal(-1) + } + if args.summary_extended: + self._time_in: decimal.Decimal = decimal.Decimal(-1) + self.max_vals = { + 'out_in': decimal.Decimal(-1), + 'at': decimal.Decimal(-1), + 'in_out': decimal.Decimal(-1), + 'in_in': decimal.Decimal(-1), + 'out_out': decimal.Decimal(-1) + } + + def feed(self, task: 'Task') -> None: + """Calculate timespans from chronological task occurrences.""" + if not self._last_finish: + self._last_start = task.time_in(self.time_unit) + self._last_finish = task.time_out(self.time_unit) + return + assert self._last_start is not None + assert self._last_finish is not None + self._time_in = task.time_in() + time_in = task.time_in(self.time_unit) + time_out = task.time_out(self.time_unit) + self.current['in_in'] = time_in - self._last_start + self.current['out_in'] = time_in - self._last_finish + self.current['in_out'] = time_out - self._last_start + self.current['out_out'] = time_out - self._last_finish + if self.args.summary_extended: + self.update_max_entries() + self._last_finish = task.time_out(self.time_unit) + self._last_start = task.time_in(self.time_unit) + + def update_max_entries(self) -> None: + """Update maximum timespans.""" + self.max_vals['in_in'] = max(self.max_vals['in_in'], self.current['in_in']) + self.max_vals['out_out'] = max(self.max_vals['out_out'], self.current['out_out']) + self.max_vals['in_out'] = max(self.max_vals['in_out'], self.current['in_out']) + if self.current['out_in'] > self.max_vals['out_in']: + self.max_vals['out_in'] = self.current['out_in'] + self.max_vals['at'] = self._time_in + +class Task: + """Handles information of a given task.""" + def __init__(self, task_id: str, tid: int, cpu: int, comm: str) -> None: + self.id = task_id + self.tid = tid + self.cpu = cpu + self.comm = comm + self.pid: Optional[int] = None + self._time_in: Optional[decimal.Decimal] = None + self._time_out: Optional[decimal.Decimal] = None + + def schedule_in_at(self, time_ns: int) -> None: + """Set schedule in time.""" + self._time_in = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + + def schedule_out_at(self, time_ns: int) -> None: + """Set schedule out time.""" + self._time_out = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + + def time_out(self, unit: str = "s") -> decimal.Decimal: + """Return schedule out time.""" + factor = TaskAnalyzer.time_uniter(unit) + return self._time_out * decimal.Decimal(factor) if self._time_out else decimal.Decimal(0) + + def time_in(self, unit: str = "s") -> decimal.Decimal: + """Return schedule in time.""" + factor = TaskAnalyzer.time_uniter(unit) + return self._time_in * decimal.Decimal(factor) if self._time_in else decimal.Decimal(0) + + def runtime(self, unit: str = "us") -> decimal.Decimal: + """Return runtime.""" + factor = TaskAnalyzer.time_uniter(unit) + if self._time_out and self._time_in: + return (self._time_out - self._time_in) * decimal.Decimal(factor) + return decimal.Decimal(0) + + def update_pid(self, pid: int) -> None: + """Update PID.""" + self.pid = pid + +class TaskAnalyzer: + """Main class for task analysis.""" + + _COLORS = { + "grey": "\033[90m", + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "violet": "\033[95m", + "reset": "\033[0m", + } + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.db: dict[str, Any] = {} + self.time_unit = "s" + if args.ns: + self.time_unit = "ns" + elif args.ms: + self.time_unit = "ms" + self._init_db() + self._check_color() + self.fd_task = sys.stdout + self.fd_sum = sys.stdout + + @contextmanager + def open_output(self, filename: str, default: Any): + """Context manager for file or stdout.""" + if filename: + with open(filename, "w", encoding="utf-8") as f: + yield f + else: + yield default + + def _init_db(self) -> None: + self.db["running"] = {} + self.db["cpu"] = {} + self.db["tid"] = {} + self.db["global"] = [] + if self.args.summary or self.args.summary_extended or self.args.summary_only: + self.db["task_info"] = {} + self.db["runtime_info"] = {} + self.db["task_info"]["pid"] = len("PID") + self.db["task_info"]["tid"] = len("TID") + self.db["task_info"]["comm"] = len("Comm") + self.db["runtime_info"]["runs"] = len("Runs") + self.db["runtime_info"]["acc"] = len("Accumulated") + self.db["runtime_info"]["max"] = len("Max") + self.db["runtime_info"]["max_at"] = len("Max At") + self.db["runtime_info"]["min"] = len("Min") + self.db["runtime_info"]["mean"] = len("Mean") + self.db["runtime_info"]["median"] = len("Median") + if self.args.summary_extended: + self.db["inter_times"] = {} + self.db["inter_times"]["out_in"] = len("Out-In") + self.db["inter_times"]["inter_at"] = len("At") + self.db["inter_times"]["out_out"] = len("Out-Out") + self.db["inter_times"]["in_in"] = len("In-In") + self.db["inter_times"]["in_out"] = len("In-Out") + + def _check_color(self) -> None: + """Check if color should be enabled.""" + if sys.stdout.isatty() and self.args.stdio_color != "never": + return + TaskAnalyzer._COLORS = {k: "" for k in TaskAnalyzer._COLORS} + + @staticmethod + def time_uniter(unit: str) -> float: + """Return time unit factor.""" + picker = {"s": 1, "ms": 1e3, "us": 1e6, "ns": 1e9} + return picker[unit] + + def _task_id(self, pid: int, cpu: int) -> str: + return f"{pid}-{cpu}" + + def _filter_non_printable(self, unfiltered: str) -> str: + filtered = "" + for char in unfiltered: + if char in string.printable: + filtered += char + return filtered + + def _prepare_fmt_precision(self) -> tuple[int, int]: + if self.args.ns: + return 0, 9 + return 3, 6 + + def _prepare_fmt_sep(self) -> tuple[str, int]: + if self.args.csv or self.args.csv_summary: + return ",", 0 + return " ", 1 + + def _fmt_header(self) -> str: + separator, fix_csv_align = self._prepare_fmt_sep() + fmt = f"{{:>{LEN_SWITCHED_IN*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_SWITCHED_OUT*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_CPU*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_PID*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_TID*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_COMM*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_RUNTIME*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_OUT_IN*fix_csv_align}}}" + if self.args.extended_times: + fmt += f"{separator}{{:>{LEN_OUT_OUT*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_IN_IN*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_IN_OUT*fix_csv_align}}}" + return fmt + + def _fmt_body(self) -> str: + separator, fix_csv_align = self._prepare_fmt_sep() + decimal_precision, time_precision = self._prepare_fmt_precision() + fmt = f"{{}}{{:{LEN_SWITCHED_IN*fix_csv_align}.{decimal_precision}f}}" + fmt += f"{separator}{{:{LEN_SWITCHED_OUT*fix_csv_align}.{decimal_precision}f}}" + fmt += f"{separator}{{:{LEN_CPU*fix_csv_align}d}}" + fmt += f"{separator}{{:{LEN_PID*fix_csv_align}d}}" + fmt += f"{separator}{{}}{{:{LEN_TID*fix_csv_align}d}}{{}}" + fmt += f"{separator}{{}}{{:>{LEN_COMM*fix_csv_align}}}" + fmt += f"{separator}{{:{LEN_RUNTIME*fix_csv_align}.{time_precision}f}}" + if self.args.extended_times: + fmt += f"{separator}{{:{LEN_OUT_IN*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_OUT_OUT*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_IN_IN*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_IN_OUT*fix_csv_align}.{time_precision}f}}{{}}" + else: + fmt += f"{separator}{{:{LEN_OUT_IN*fix_csv_align}.{time_precision}f}}{{}}" + return fmt + + def _print_header(self) -> None: + fmt = self._fmt_header() + header = ["Switched-In", "Switched-Out", "CPU", "PID", "TID", "Comm", + "Runtime", "Time Out-In"] + if self.args.extended_times: + header += ["Time Out-Out", "Time In-In", "Time In-Out"] + self.fd_task.write(fmt.format(*header) + "\n") + + def _print_task_finish(self, task: Task) -> None: + c_row_set = "" + c_row_reset = "" + out_in: Any = -1 + out_out: Any = -1 + in_in: Any = -1 + in_out: Any = -1 + fmt = self._fmt_body() + + if str(task.tid) in self.args.highlight_tasks_map: + c_row_set = TaskAnalyzer._COLORS[self.args.highlight_tasks_map[str(task.tid)]] + c_row_reset = TaskAnalyzer._COLORS["reset"] + if task.comm in self.args.highlight_tasks_map: + c_row_set = TaskAnalyzer._COLORS[self.args.highlight_tasks_map[task.comm]] + c_row_reset = TaskAnalyzer._COLORS["reset"] + + c_tid_set = "" + c_tid_reset = "" + if task.pid == task.tid: + c_tid_set = TaskAnalyzer._COLORS["grey"] + c_tid_reset = TaskAnalyzer._COLORS["reset"] + + if task.tid in self.db["tid"]: + last_tid_task = self.db["tid"][task.tid][-1] + timespan_gap_tid = Timespans(self.args, self.time_unit) + timespan_gap_tid.feed(last_tid_task) + timespan_gap_tid.feed(task) + out_in = timespan_gap_tid.current['out_in'] + out_out = timespan_gap_tid.current['out_out'] + in_in = timespan_gap_tid.current['in_in'] + in_out = timespan_gap_tid.current['in_out'] + + if self.args.extended_times: + line_out = fmt.format(c_row_set, task.time_in(), task.time_out(), task.cpu, + task.pid, c_tid_set, task.tid, c_tid_reset, c_row_set, task.comm, + task.runtime(self.time_unit), out_in, out_out, in_in, in_out, + c_row_reset) + "\n" + else: + line_out = fmt.format(c_row_set, task.time_in(), task.time_out(), task.cpu, + task.pid, c_tid_set, task.tid, c_tid_reset, c_row_set, task.comm, + task.runtime(self.time_unit), out_in, c_row_reset) + "\n" + self.fd_task.write(line_out) + + def _record_cleanup(self, _list: list[Any]) -> list[Any]: + if not self.args.summary and len(_list) > 1: + return _list[len(_list) - 1:] + return _list + + def _record_by_tid(self, task: Task) -> None: + tid = task.tid + if tid not in self.db["tid"]: + self.db["tid"][tid] = [] + self.db["tid"][tid].append(task) + self.db["tid"][tid] = self._record_cleanup(self.db["tid"][tid]) + + def _record_by_cpu(self, task: Task) -> None: + cpu = task.cpu + if cpu not in self.db["cpu"]: + self.db["cpu"][cpu] = [] + self.db["cpu"][cpu].append(task) + self.db["cpu"][cpu] = self._record_cleanup(self.db["cpu"][cpu]) + + def _record_global(self, task: Task) -> None: + self.db["global"].append(task) + self.db["global"] = self._record_cleanup(self.db["global"]) + + def _handle_task_finish(self, tid: int, cpu: int, time_ns: int, pid: int) -> None: + if tid == 0: + return + _id = self._task_id(tid, cpu) + if _id not in self.db["running"]: + return + task = self.db["running"][_id] + task.schedule_out_at(time_ns) + task.update_pid(pid) + del self.db["running"][_id] + + if not self._limit_filtered(tid, pid, task.comm) and not self.args.summary_only: + self._print_task_finish(task) + self._record_by_tid(task) + self._record_by_cpu(task) + self._record_global(task) + + def _handle_task_start(self, tid: int, cpu: int, comm: str, time_ns: int) -> None: + if tid == 0: + return + if tid in self.args.tid_renames: + comm = self.args.tid_renames[tid] + _id = self._task_id(tid, cpu) + if _id in self.db["running"]: + return + task = Task(_id, tid, cpu, comm) + task.schedule_in_at(time_ns) + self.db["running"][_id] = task + + def _limit_filtered(self, tid: int, pid: int, comm: str) -> bool: + """Filter tasks based on CLI arguments.""" + if self.args.filter_tasks: + if (str(tid) in self.args.filter_tasks or + str(pid) in self.args.filter_tasks or + comm in self.args.filter_tasks): + return True + return False + if self.args.limit_to_tasks: + if (str(tid) in self.args.limit_to_tasks or + str(pid) in self.args.limit_to_tasks or + comm in self.args.limit_to_tasks): + return False + return True + return False + + def _is_within_timelimit(self, time_ns: int) -> bool: + if not self.args.time_limit: + return True + time_s = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + lower_bound, upper_bound = self.args.time_limit.split(":") + if lower_bound and time_s < decimal.Decimal(lower_bound): + return False + if upper_bound and time_s > decimal.Decimal(upper_bound): + return False + return True + + def process_event(self, sample: perf.sample_event) -> None: + """Process sched:sched_switch events.""" + if "sched:sched_switch" not in str(sample.evsel): + return + + time_ns = sample.sample_time + if not self._is_within_timelimit(time_ns): + return + + # Access tracepoint fields directly from sample object + try: + prev_pid = sample.prev_pid + next_pid = sample.next_pid + next_comm = sample.next_comm + common_cpu = sample.sample_cpu + except AttributeError: + # Fallback or ignore if fields are not available + return + + next_comm = self._filter_non_printable(next_comm) + + # Task finish for previous task + self._handle_task_finish(prev_pid, common_cpu, time_ns, prev_pid) + # Task start for next task + self._handle_task_start(next_pid, common_cpu, next_comm, time_ns) + + def print_summary(self) -> None: + """Calculate and print summary.""" + if not (self.args.summary or self.args.summary_extended or self.args.summary_only): + return + + # Simplified summary logic for brevity, full logic can be ported if needed + print("\nSummary (Simplified)", file=self.fd_sum) + if self.args.summary_extended: + print("Inter Task Times", file=self.fd_sum) + # ... port full Summary class logic here ... + + def _run_file(self) -> None: + if not self.args.summary_only: + self._print_header() + + session = perf.session(perf.data(self.args.input), sample=self.process_event) + session.process_events() + + self.print_summary() + + def _run_live(self) -> None: + if not self.args.summary_only: + self._print_header() + + cpus = perf.cpu_map() + threads = perf.thread_map(-1) + evlist = perf.parse_events("sched:sched_switch", cpus, threads) + evlist.config() + + evlist.open() + evlist.mmap() + evlist.enable() + + print("Live mode started. Press Ctrl+C to stop.", file=sys.stderr) + try: + while True: + evlist.poll(timeout=-1) + for cpu in cpus: + while True: + event = evlist.read_on_cpu(cpu) + if not event: + break + if not isinstance(event, perf.sample_event): + continue + self.process_event(event) + except KeyboardInterrupt: + print("\nStopping live mode...", file=sys.stderr) + finally: + evlist.close() + self.print_summary() + + def run(self) -> None: + """Run the session.""" + with self.open_output(self.args.csv, sys.stdout) as fd_task: + with self.open_output(self.args.csv_summary, sys.stdout) as fd_sum: + self.fd_task = fd_task + self.fd_sum = fd_sum + if self.args.csv: + fd_task.write("Comm;Out-Out;\n") + if self.args.csv_summary: + fd_sum.write("Comm;Out-Out;\n") + + if not os.path.exists(self.args.input) and self.args.input == "perf.data": + self._run_live() + else: + self._run_file() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Analyze tasks behavior") + parser.add_argument("-i", "--input", default="perf.data", help="Input file name") + parser.add_argument("--time-limit", default="", help="print tasks only in time window") + parser.add_argument("--summary", action="store_true", + help="print additional runtime information") + parser.add_argument("--summary-only", action="store_true", + help="print only summary without traces") + parser.add_argument("--summary-extended", action="store_true", + help="print extended summary") + parser.add_argument("--ns", action="store_true", help="show timestamps in nanoseconds") + parser.add_argument("--ms", action="store_true", help="show timestamps in milliseconds") + parser.add_argument("--extended-times", action="store_true", + help="Show elapsed times between schedule in/out") + parser.add_argument("--filter-tasks", default="", help="filter tasks by tid, pid or comm") + parser.add_argument("--limit-to-tasks", default="", help="limit output to selected tasks") + parser.add_argument("--highlight-tasks", default="", help="colorize special tasks") + parser.add_argument("--rename-comms-by-tids", default="", help="rename task names by using tid") + parser.add_argument("--stdio-color", default="auto", choices=["always", "never", "auto"], + help="configure color output") + parser.add_argument("--csv", default="", help="Write trace to file") + parser.add_argument("--csv-summary", default="", help="Write summary to file") + + args = parser.parse_args() + args.tid_renames = {} + args.highlight_tasks_map = {} + args.filter_tasks = args.filter_tasks.split(",") if args.filter_tasks else [] + args.limit_to_tasks = args.limit_to_tasks.split(",") if args.limit_to_tasks else [] + + if args.rename_comms_by_tids: + for item in args.rename_comms_by_tids.split(","): + tid, name = item.split(":") + args.tid_renames[int(tid)] = name + + if args.highlight_tasks: + for item in args.highlight_tasks.split(","): + parts = item.split(":") + if len(parts) == 1: + parts.append("red") + key, color = parts[0], parts[1] + args.highlight_tasks_map[key] = color + + analyzer = TaskAnalyzer(args) + analyzer.run() + +if __name__ == "__main__": + main()
tools/perf/tests/shell/test_task_analyzer.sh +41 -30
150 lines not on this page · raw message
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog
[email protected]
AI reviewRe: [PATCH v1 46/58] perf task-analyzer: Port task-analyzer to use python module
Sashiko AI review found 7 potential issue(s): - [High] Incomplete implementation of the summary feature - [High] Unbounded memory growth in live mode with summary enabled - [Medium] Process ID…
Sashiko AI review found 7 potential issue(s):
- [High] Incomplete implementation of the summary feature
- [High] Unbounded memory growth in live mode with summary enabled
- [Medium] Process ID (TGID) is incorrectly tracked as Thread ID (TID)
- [Medium] ANSI color escape codes are injected into CSV output
- [Medium] Conflicting headers and delimiters corrupt the CSV output
- [Medium] Mutually exclusive filter arguments (limit-to-tasks is ignored)
- [Medium] Task history lists are erroneously truncated for extended/only summary options --
commit 5635b811218ca3b0adbc654f9a41d8a9897ae75b
Author: Ian Rogers <[email protected]>
perf task-analyzer: Port task-analyzer to use python module
This commit ports task-analyzer.py to the perf.session python module and refactors it into a class-based architecture. It adds support for live mode and file mode while keeping previous command-line options.
Quoted 5 lines
tools/perf/python/task-analyzer.py new file +0
-0
[ ... ]
tools/perf/python/task-analyzer.py line 196
Quoted 5 lines
tools/perf/python/task-analyzer.py new file +5
-0+ def _check_color(self) -> None: + """Check if color should be enabled.""" + if sys.stdout.isatty() and self.args.stdio_color != "never": + return + TaskAnalyzer._COLORS = {k: "" for k in TaskAnalyzer._COLORS}
Does this inject ANSI color escape codes into the CSV output?
When using --csv, sys.stdout might still be attached to a terminal even though the tabular data is written to a file. This would cause the check to pass and corrupt the CSV fields with color codes.
[ ... ]
tools/perf/python/task-analyzer.py line 313
Quoted 4 lines
tools/perf/python/task-analyzer.py new file +4
-0+ def _record_cleanup(self, _list: list[Any]) -> list[Any]: + if not self.args.summary and len(_list) > 1: + return _list[len(_list) - 1:] + return _list
Should this check include self.args.summary_extended and self.args.summary_only?
If those flags are used without --summary, the arrays are prematurely truncated, which might break calculations that rely on the full history.
Could this also cause unbounded memory growth in live mode?
If self.args.summary is set, _record_cleanup() doesn't truncate the lists. Since _run_live() runs an infinite loop, it looks like the self.db arrays will grow forever and eventually exhaust memory.
[ ... ]
Quoted 5 lines
tools/perf/python/task-analyzer.py new file +5
-0+ def _limit_filtered(self, tid: int, pid: int, comm: str) -> bool: + """Filter tasks based on CLI arguments.""" + if self.args.filter_tasks: + if (str(tid) in self.args.filter_tasks or + str(pid) in self.args.filter_tasks or
tools/perf/python/task-analyzer.py line 376
Quoted 10 lines
tools/perf/python/task-analyzer.py new file +10
-0+ comm in self.args.filter_tasks): + return True + return False + if self.args.limit_to_tasks: + if (str(tid) in self.args.limit_to_tasks or + str(pid) in self.args.limit_to_tasks or + comm in self.args.limit_to_tasks): + return False + return True + return False
Are --filter-tasks and --limit-to-tasks meant to be mutually exclusive?
If self.args.filter_tasks is specified but no match is found, the function returns False early, meaning the limit_to_tasks check is unreachable.
[ ... ]
Quoted 13 lines
tools/perf/python/task-analyzer.py new file +13
-0+ def process_event(self, sample: perf.sample_event) -> None: + """Process sched:sched_switch events.""" + if "sched:sched_switch" not in str(sample.evsel): + return + + time_ns = sample.sample_time + if not self._is_within_timelimit(time_ns): + return + + # Access tracepoint fields directly from sample object + try: + prev_pid = sample.prev_pid + next_pid = sample.next_pid
tools/perf/python/task-analyzer.py line 411
Quoted 10 lines
tools/perf/python/task-analyzer.py new file +10
-0+ next_comm = sample.next_comm + common_cpu = sample.sample_cpu + except AttributeError: + # Fallback or ignore if fields are not available + return + + next_comm = self._filter_non_printable(next_comm) + + # Task finish for previous task + self._handle_task_finish(prev_pid, common_cpu, time_ns, prev_pid)
Are we incorrectly tracking the Thread ID (TID) as the Process ID (TGID) here?
sample.prev_pid corresponds to the kernel's thread ID, but it's passed as both the tid and pid arguments to _handle_task_finish(). This loses the actual Process ID information and breaks the PID column output.
[ ... ]
tools/perf/python/task-analyzer.py line 424
Quoted 10 lines
tools/perf/python/task-analyzer.py new file +10
-0+ def print_summary(self) -> None: + """Calculate and print summary.""" + if not (self.args.summary or self.args.summary_extended or self.args.summary_only): + return + + # Simplified summary logic for brevity, full logic can be ported if needed + print("\nSummary (Simplified)", file=self.fd_sum) + if self.args.summary_extended: + print("Inter Task Times", file=self.fd_sum) + # ... port full Summary class logic here ...
Does this intentionally omit the full summary logic?
The --summary flags seem to be broken compared to the legacy version because the actual calculation and output are missing.
[ ... ]
Quoted 5 lines
tools/perf/python/task-analyzer.py new file +5
-0+ def run(self) -> None: + """Run the session.""" + with self.open_output(self.args.csv, sys.stdout) as fd_task: + with self.open_output(self.args.csv_summary, sys.stdout) as fd_sum: + self.fd_task = fd_task
tools/perf/python/task-analyzer.py line 480
Quoted 10 lines
tools/perf/python/task-analyzer.py new file +10
-0+ self.fd_sum = fd_sum + if self.args.csv: + fd_task.write("Comm;Out-Out;\n") + if self.args.csv_summary: + fd_sum.write("Comm;Out-Out;\n") + + if not os.path.exists(self.args.input) and self.args.input == "perf.data": + self._run_live() + else: + self._run_file()
Does this corrupt the CSV file by writing two conflicting headers?
This writes a hardcoded header with a semicolon delimiter, but immediately after, _run_file() calls _print_header() which writes comma-separated headers. This leaves the file with multiple contradictory headers.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=46
Ian Rogers
reviewerPATCH 47/58 perf failed-syscalls: Port failed-syscalls to use python module
In reply to Ian RogersPort the legacy Perl script failed-syscalls.pl to a python script using the perf module in tools/perf/python.
The new script uses a class-based architecture and leverages the perf.session API for event processing, making it a standalone script that reads perf.data files.
It filters for sys_exit events, checks for failed syscalls (where return value ret < 0), and aggregates counts per command name.
Complications:
- The script is designed for file-based processing using perf.session.
- pylint warns about the module name not being snake_case, but it is kept for consistency with the original script name.
--- tools/perf/python/failed-syscalls.py | 75 ++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 tools/perf/python/failed-syscalls.py
tools/perf/python/failed-syscalls.py new file +75 -0
@@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Failed system call counts.""" + +import argparse +from collections import defaultdict +import sys +from typing import Optional +import perf + +class FailedSyscalls: + """Tracks and displays failed system call totals.""" + def __init__(self, comm: Optional[str] = None) -> None: + self.failed_syscalls: dict[str, int] = defaultdict(int) + self.for_comm = comm + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process sys_exit events.""" + event_name = str(sample.evsel) + if "sys_exit" not in event_name: + return + + try: + ret = sample.ret + except AttributeError: + return + + if ret >= 0: + return + + pid = sample.sample_pid + assert self.session is not None + try: + comm = self.session.process(pid).comm() + except Exception: # pylint: disable=broad-except + comm = "unknown" + + if self.for_comm and comm != self.for_comm: + return + + self.failed_syscalls[comm] += 1 + + def print_totals(self) -> None: + """Print summary table.""" + print("\nfailed syscalls by comm:\n") + print(f"{'comm':<20s} {'# errors':>10s}") + print(f"{'-'*20} {'-'*10}") + + for comm, val in sorted(self.failed_syscalls.items(), + key=lambda kv: (kv[1], kv[0]), reverse=True): + print(f"{comm:<20s} {val:10d}") + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + self.session.process_events() + self.print_totals() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace failed syscalls") + parser.add_argument("comm", nargs="?", help="Filter by command name") + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + args = parser.parse_args() + + analyzer = FailedSyscalls(args.comm) + try: + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main()
Signature · 1 line
2.54.0.rc1.513.gad8abe7a5a-goog