Start KernelShark v2 transformation
cover Start KernelShark v2 transformation
In this patch-set we are starting the introduction of the changes in the C API of libkshark. The goal of this transformation is to turn KernelShark into a general purpose toolkit for analysis of tracing data that is no longer strongly coupled to the Ftrace/trace-cmd data format. At the end of the transformation KernelShark will be able to merge and simultaneously visualize data from different tracers, recorded in different data formats. It will also significantly extend the possibilities for user customization.
This patch-set transforms the way KernelShark processes the raw tracing data. Later, in a consecutive patch-sets we will introduce the changes in the visualization instruments and in the KernelShark GUI itself.
Yordan Karadzhov (VMware) (15):
kernel-shark: split kernel-shark from trace-cmd repo
kernel-shark: Version 1.2.0
kernel-shark: Start introducing KernelShark 2.0
kernel-shark: Use only signed types in kshark_entry
kernel-shark: Introduce libkshark-hash
kernel-shark: Introduce Data streams
kernel-shark: Add stream_id to kshark_entry
kernel-shark: Integrate the stream definitions with the C API
kernel-shark: Provide merging of multiple data streams
kernel-shark: Integrate the stream definitions with data model
kernel-shark: Use only signed types for model defs
kernel-shark: Add ksmodel_get_bin()
kernel-shark: Protect ksmodel_set_in_range_bining()
kernel-shark: Add methods for time calibration
kernel-shark: Integrate streams with libkshark-configio
CMakeLists.txt | 12 +- build/FindTraceCmd.cmake | 8 +- build/deff.h.cmake | 14 +- examples/CMakeLists.txt | 34 +- examples/configio.c | 20 +- examples/datafilter.c | 67 +- examples/datahisto.c | 38 +- examples/dataload.c | 22 +- examples/multibufferload.c | 60 + src/CMakeLists.txt | 6 +- src/libkshark-collection.c | 121 +- src/libkshark-configio.c | 1175 +++++++++++++++--- src/libkshark-hash.c | 213 ++++ src/libkshark-model.c | 154 ++- src/libkshark-model.h | 55 +- src/libkshark-plugin.c | 597 ++++++++-- src/libkshark-plugin.h | 279 +++-- src/libkshark-tepdata.c | 1754 +++++++++++++++++++++++++++ src/libkshark-tepdata.h | 100 ++ src/libkshark.c | 2314 ++++++++++++++++-------------------- src/libkshark.h | 795 +++++++++++-- 21 files changed, 5890 insertions(+), 1948 deletions(-) create mode 100644 examples/multibufferload.c create mode 100644 src/libkshark-hash.c create mode 100644 src/libkshark-tepdata.c create mode 100644 src/libkshark-tepdata.h
Signature · 1 line
2.25.1
PATCH 1/15 kernel-shark: split kernel-shark from trace-cmd repo
Kernel-shark is no longer a directory in the trace-cmd repository. It is now a stand alone repository, hence we should not try to search for the trace-cmd libraries in the original directory tree of the trace-cmd repo.
--- build/FindTraceCmd.cmake | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
build/FindTraceCmd.cmake +1 -7
@@ -18,7 +18,6 @@ if (CMAKE_BUILD_TYPE MATCHES Debug) find_program(TRACECMD_EXECUTABLE NAMES trace-cmd PATHS $ENV{TRACE_CMD}/tracecmd/ - ${CMAKE_SOURCE_DIR}/../tracecmd/ NO_DEFAULT_PATH) endif (CMAKE_BUILD_TYPE MATCHES Debug)
@@ -31,26 +30,22 @@ endif (NOT TRACECMD_EXECUTABLE) find_path(TRACECMD_INCLUDE_DIR NAMES trace-cmd/trace-cmd.h PATHS $ENV{TRACE_CMD}/include/ - ${CMAKE_SOURCE_DIR}/../include/ NO_DEFAULT_PATH) + find_path(TRACEFS_INCLUDE_DIR NAMES tracefs/tracefs.h PATHS $ENV{TRACE_CMD}/include/ - ${CMAKE_SOURCE_DIR}/../include/ NO_DEFAULT_PATH) find_library(TRACECMD_LIBRARY NAMES trace-cmd/libtracecmd.a PATHS $ENV{TRACE_CMD}/lib/ - ${CMAKE_SOURCE_DIR}/../lib/ NO_DEFAULT_PATH) find_library(TRACEFS_LIBRARY NAMES tracefs/libtracefs.a PATHS $ENV{TRACE_CMD}/lib/ - ${CMAKE_SOURCE_DIR}/../lib/ NO_DEFAULT_PATH) find_library(TRACEEVENT_LIBRARY NAMES traceevent/libtraceevent.a PATHS $ENV{TRACE_CMD}/lib/ - ${CMAKE_SOURCE_DIR}/../lib/ NO_DEFAULT_PATH) # If not found, search in the default system paths. Note that if the previous
@@ -94,7 +89,6 @@ ELSE (TRACEFS_FOUND) ENDIF (TRACEFS_FOUND) - IF (TRACEEVENT_LIBRARY) SET(TRACEEVENT_FOUND TRUE)
Signature · 1 line
2.25.1
PATCH 2/15 kernel-shark: Version 1.2.0
In reply to Yordan Karadzhov (VMware)--- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
CMakeLists.txt +1 -1
@@ -6,7 +6,7 @@ project(kernel-shark) set(KS_APP_NAME "kernelshark") set(KS_VERSION_MAJOR 1) -set(KS_VERSION_MINOR 1) +set(KS_VERSION_MINOR 2) set(KS_VERSION_PATCH 0) set(KS_VERSION_STRING ${KS_VERSION_MAJOR}.${KS_VERSION_MINOR}.${KS_VERSION_PATCH}) message("\n project: Kernel Shark: (version: ${KS_VERSION_STRING})\n")
Signature · 1 line
2.25.1
PATCH 3/15 kernel-shark: Start introducing KernelShark 2.0
In reply to Yordan Karadzhov (VMware)This is the very first patch from a series of patches that will introduce KernelShark 2.0. The new version implies significant changes in the low level C API, used by the KernelShark GUI. As a consequence of the API modifications, significant changes are needed in every source file of the project. Because, we want to avoid having a huge and ugly single commit that will be extremely hard to review, we will introduce a regression. We will first disable the compilation of all sub-components of KernelShark, except the API itself. This is done by bypassing the search of the necessary third party components (OpenGL and Qt). Later we will introduce the changes of the API in several sub-steps and will re-enable one by one the modified components of the GUI.
--- CMakeLists.txt | 10 ++++----- build/deff.h.cmake | 14 ++----------- examples/CMakeLists.txt | 46 ++++++++++++++++++++--------------------- src/CMakeLists.txt | 6 +++--- 4 files changed, 33 insertions(+), 43 deletions(-)
CMakeLists.txt +5 -5
@@ -26,12 +26,12 @@ include(${KS_DIR}/build/FindJSONC.cmake) find_package(Doxygen) -set(OpenGL_GL_PREFERENCE LEGACY) -find_package(OpenGL) -find_package(GLUT) +# set(OpenGL_GL_PREFERENCE LEGACY) +# find_package(OpenGL) +# find_package(GLUT) -find_package(Qt5Widgets 5.7.1) -find_package(Qt5Network) +# find_package(Qt5Widgets 5.7.1) +# find_package(Qt5Network) if (Qt5Widgets_FOUND) message(STATUS "Found Qt5Widgets: (version ${Qt5Widgets_VERSION})")
build/deff.h.cmake +2 -12
@@ -23,17 +23,7 @@ /** "pkexec" executable. */ #cmakedefine DO_AS_ROOT "@DO_AS_ROOT@" -#ifdef __cplusplus - - #include <QString> - - /** - * String containing semicolon-separated list of plugin names. - * The plugins to be loaded when KernelShark starts are tagged - * with "default". - */ - const QString plugins = "@PLUGINS@"; - -#endif /* __cplusplus */ +/** Semicolon-separated list of plugin names. */ +#define KS_BUILTIN_PLUGINS "@PLUGINS@" #endif // _KS_CONFIG_H
examples/CMakeLists.txt +23 -23
49 lines not on this page · raw message
src/CMakeLists.txt +3 -3
@@ -2,9 +2,9 @@ message("\n src ...") message(STATUS "libkshark") add_library(kshark SHARED libkshark.c - libkshark-model.c +# libkshark-model.c libkshark-plugin.c - libkshark-configio.c +# libkshark-configio.c libkshark-collection.c) target_link_libraries(kshark ${TRACECMD_LIBRARY}
@@ -99,7 +99,7 @@ if (Qt5Widgets_FOUND AND Qt5Network_FOUND) endif (Qt5Widgets_FOUND AND Qt5Network_FOUND) -add_subdirectory(plugins) +# add_subdirectory(plugins) find_program(DO_AS_ROOT pkexec)
Signature · 1 line
2.25.1
Re: [PATCH 03/15] kernel-shark: Start introducing KernelShark 2.0
Quoted 4 lines · On Tue, 29 Sep 2020 16:41:11 +0300 "Yordan Karadzhov (VMware)"…
This is the very first patch from a series of patches that will introduce KernelShark 2.0. The new version implies significant changes in the low level C API, used by the KernelShark GUI. As a consequence of the API modifications, significant changes are
Quoted 8 lines
needed in every source file of the project. Because, we want to avoid having a huge and ugly single commit that will be extremely hard to review, we will introduce a regression. We will first disable the compilation of all sub-components of KernelShark, except the API itself. This is done by bypassing the search of the necessary third party components (OpenGL and Qt). Later we will introduce the changes of the API in several sub-steps and will re-enable one by one the modified components of the GUI.
OK, this isn't what I was thinking. It's no different than just letting the code break the build, because it doesn't allow use to bisect through this either.
-- Steve
PATCH 4/15 kernel-shark: Use only signed types in kshark_entry
In reply to Yordan Karadzhov (VMware)Using uint64_t for the value of the offset was just wrong. According to the POSIX standard off_t is a signed integer type with unspecified size. Here we stick to a 64 bit integer, because this size guaranties optimal packing of the kshark_entry structure.
Using unsigned values for the timestamps is also a source of problems and has been a reason for the introduction of multiple bugs in the past. In principal the value of the timestamps cannot be negative. However, this value must have the same type as the values used to define the state of the visualization model, like the range of the model or the size of the bin. The model state definitions should not take negative values as well, however their values are recalculated automatically when the user browses the data and those calculations may result in negative values in some corner cases. Because of this it is better to use a signed integer type and treat the negative values as an indicator of an error rather than have the negative result of the calculations casted into unsigned type which results into unpredictable behavior of the model.
--- src/libkshark.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
src/libkshark.h +2 -2
@@ -61,7 +61,7 @@ struct kshark_entry {int32_t event_id;/** The offset into the trace file, used to find the record. */ -uint64_t offset; +int64_t offset;/** * The time of the record in nano seconds. The value is taken from
@@ -69,7 +69,7 @@ struct kshark_entry {* dependent. The time usually is the timestamp from when the system * started. */ -uint64_t ts; +int64_t ts; };/** Size of the task's hash table. */
Signature · 1 line
2.25.1
PATCH 5/15 kernel-shark: Introduce libkshark-hash
In reply to Yordan Karadzhov (VMware)So far KernelShark have been using an implementation of a hash table from trace-cmd/include/trace-cmd/trace-filter-hash.h. However it turns that KernelShark is the only user of trace-filter-hash, which means that it make more sense to make this implementation of the hash table part of KernelShark. In this patch we adapt the original trace-cmd implementation and change the naming convention used.
--- src/CMakeLists.txt | 1 + src/libkshark-hash.c | 213 +++++++++++++++++++++++++++++++++++++++++++ src/libkshark.h | 47 ++++++++++ 3 files changed, 261 insertions(+) create mode 100644 src/libkshark-hash.c
src/CMakeLists.txt +1 -0
@@ -2,6 +2,7 @@ message("\n src ...") message(STATUS "libkshark") add_library(kshark SHARED libkshark.c + libkshark-hash.c # libkshark-model.c libkshark-plugin.c # libkshark-configio.c
src/libkshark-hash.c new file +213 -0
213 lines not on this page · raw message
src/libkshark.h +47 -0
59 lines not on this page · raw message
Signature · 1 line
2.25.1
Re: [PATCH 05/15] kernel-shark: Introduce libkshark-hash
src/libkshark-hash.c line 62
Quoted 10 lines · On Tue, 29 Sep 2020 16:41:13 +0300 "Yordan Karadzhov (VMware)"…
src/libkshark-hash.c new file +10
-0+/** +* Create new hash table of Ids. +*/ +struct kshark_hash_id *kshark_hash_id_alloc (size_t n_bits) +{ +struct kshark_hash_id *hash; +size_t size; + + hash =calloc (1 ,sizeof (*hash)); +assert (hash);
For failed memory, is the general plan to just crash? (assert) or return NULL?
src/libkshark-hash.c line 68
Quoted 6 lines
src/libkshark-hash.c new file +6
-0+ + hash->n_bits = n_bits; + hash->count =0 ; + + size =hash_size (hash); + hash->hash =calloc (size,sizeof (*hash->hash));
In either case, this needs to be checked too.
Quoted 21 lines
src/libkshark-hash.c new file +15
-0+ +return hash; +} + +/** Free the hash table of Ids. */ +void kshark_hash_id_free (struct kshark_hash_id *hash) +{ +if (!hash) +return ; + +kshark_hash_id_clear (hash); +free (hash->hash); +free (hash); +} +src/libkshark.h +0
-0
src/libkshark.h extern "C" { · line 36
Quoted 4 lines
src/libkshark.h +1
-0@@ -33,6 +33,7 @@ extern "C" {// KernelShark #include "libkshark-plugin.h" +
Extra white space.
src/libkshark.h struct kshark_entry { · line 76
Quoted 8 lines
src/libkshark.h +1
-0/** * Kernel Shark entry contains all information from one trace record needed * in order to visualize the time-series of trace records. The part of the @@ -72,6 +73,52 @@ struct kshark_entry {int64_t ts; }; +/** Size of the task'c hash table in terms of bits being used by the key. */
"task'c" ?
-- Steve
Quoted 32 lines
src/libkshark.h +32
-0+#define KS_TASK_HASH_NBITS 16 + +/** Size of the hash table of Ids in terms of bits being used by the key. */ +#define KS_FILTER_HASH_NBITS 8 + +/** A bucket for the hash table of integer Id numbers (kshark_hash_id). */ +struct kshark_hash_id_item { +/** Pointer to the Id in this bucket. */ +struct kshark_hash_id_item *next; + +/** The Id value. */ +int id; +}; + +/** +* Hash table of integer Id numbers. To be used for fast filter of trace +* entries. +*/ +struct kshark_hash_id { +/** Array of buckets. */ +struct kshark_hash_id_item **hash; + +/** The number of Ids in the table. */ +size_t count; + +/** +* The number of bits used by the hashing function. +* Note that the number of buckets in the table if given by +* 1 << n_bits. +*/ +size_t n_bits; +};
PATCH 6/15 kernel-shark: Introduce Data streams
In reply to Yordan Karadzhov (VMware)With the help of Data stream, KernelShark will be able to load and merge multiple trace files (streams). Each stream can have different plugins or filters, registered for it, which means that the raw trace data of the streams can have different formats, and will allow for a great degree of customization of the provided data visualization. In this patch we only provide the basic definitions. The actual integration of the Data streams into the C API of KernelShark will happen in the following patches.
--- src/libkshark.h | 232 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+)
src/libkshark.h +232 -0
@@ -119,6 +119,238 @@ void kshark_hash_id_free(struct kshark_hash_id *hash);int *kshark_hash_ids (struct kshark_hash_id *hash); +/** +* Timestamp calibration function type. To be user for system clock +* calibration. +*/ +typedef void (*time_calib_func ) (int64_t *,int64_t *); + +struct kshark_data_stream ; + +/** A function type to be used to initialize the interface of the data stream. */ +typedef int (*interface_init_func ) (struct kshark_data_stream *, +const char *); + +/** A function type to be used to initialize the interface of the data stream. */ +typedef int (*interface_close_func ) (struct kshark_data_stream *, +const char *); + +/** A function type to be used by the method interface of the data stream. */ +typedef char *(*stream_get_str_func ) (struct kshark_data_stream *, +const struct kshark_entry *); + +/** A function type to be used by the method interface of the data stream. */ +typedef char *(*stream_get_c_str_func ) (struct kshark_data_stream *, +const struct kshark_entry *); + +/** A function type to be used by the method interface of the data stream. */ +typedef const int (*stream_get_int_func ) (struct kshark_data_stream *, +const struct kshark_entry *); + +/** A function type to be used by the method interface of the data stream. */ +typedef int (*stream_find_id_func ) (struct kshark_data_stream *, +const char *); + +/** A function type to be used by the method interface of the data stream. */ +typedef int *(*stream_get_ids_func ) (struct kshark_data_stream *); + +/** A function type to be used by the method interface of the data stream. */ +typedef int (*stream_get_names_func ) (struct kshark_data_stream *, +const struct kshark_entry *, +char ***); + +/** Event field format identifier. */ +typedef enum kshark_event_field_format { +/** A field of unknown type. */ +KS_INVALIDE_FIELD , + +/** Integer number */ +KS_INTEGER_FIELD , + +/** Floating-point number */ +KS_FLOAT_FIELD +}kshark_event_field_format ; + +/** A function type to be used by the method interface of the data stream. */ +typedef kshark_event_field_format +(*stream_event_field_type ) (struct kshark_data_stream *, +const struct kshark_entry *, +const char *); + +/** A function type to be used by the method interface of the data stream. */ +typedef const int (*stream_read_event_field ) (struct kshark_data_stream *, +const struct kshark_entry *, +const char *, +int64_t *); + +/** A function type to be used by the method interface of the data stream. */ +typedef const int (*stream_read_record_field ) (struct kshark_data_stream *, +void *, +const char *, +int64_t *); + +struct kshark_context ; + +/** A function type to be used by the method interface of the data stream. */ +typedef ssize_t (*load_entries_func ) (struct kshark_data_stream *, +struct kshark_context *, +struct kshark_entry ***); + +/** A function type to be used by the method interface of the data stream. */ +typedef ssize_t (*load_matrix_func ) (struct kshark_data_stream *, +struct kshark_context *, +int16_t **cpu_array, +int32_t **pid_array, +int32_t **event_array, +int64_t **offset_array, +int64_t **ts_array); + +/** Data format identifier. */ +typedef enum kshark_data_format { +/** A data of unknown type. */ +KS_INVALIDE_DATA , + +/** Ftrace data. */ +KS_TEP_DATA , + +/** VMware SchedTrace data. */ +KS_VMW_ST_DATA , +}kshark_data_format ; + +/** +* Structure representing the interface of methods used to operate over +* the data from a given stream. +*/ +struct kshark_data_stream_interface { +/** Method used to retrieve the Process Id of the entry. */ +stream_get_int_func get_pid; + +/** Method used to retrieve the Event Id of the entry. */ +stream_get_int_func get_event_id; + +/** Method used to retrieve the Event name of the entry. */ +stream_get_c_str_func get_event_name; + +/** Method used to retrieve the Task name of the entry. */ +stream_get_c_str_func get_task; + +/** Method used to retrieve the Latency of the entry. */ +stream_get_c_str_func get_latency; + +/** Method used to retrieve the Info string of the entry. */ +stream_get_c_str_func get_info; + +/** Method used to retrieve Id of the Event from its name. */ +stream_find_id_func find_event_id; + +/** Method used to retrieve the array of Ids of all Events. */ +stream_get_ids_func get_all_event_ids; + +/** Method used to dump the entry's content to string. */ +stream_get_str_func dump_entry; + +/** +* Method used to retrieve the array of all field names of a given +* event. +*/ +stream_get_names_func get_all_field_names; + +/** Method used to access the type of an event's data field. */ +stream_event_field_type get_event_field_type; + +/** Method used to access the value of an event's data field. */ +stream_read_event_field read_event_field_int64; + +/** Method used to access the value of an event's data field. */ +stream_read_record_field read_record_field_int64; + +/** Method used to load the data in the form of entries. */ +load_entries_func load_entries; + +/** Method used to load the data in matrix form. */ +load_matrix_func load_matrix; + +/** Generic data handle. */ +void *handle; +}; + +/** Structure representing a stream of trace data. */ +struct kshark_data_stream { +/** Data stream identifier. */ +uint8_t stream_id; + +/** The number of CPUs presented in this data stream. */ +int n_cpus; + +/** +* The number of distinct event types presented in this data stream. +*/ +int n_events; + +/** The Process Id of the Idle task. */ +int idle_pid; + +/** Trace data file pathname. */ +char *file; + +/** Stream name. */ +char *name; + +/** System clock calibration function. */ +time_calib_func calib; + +/** An array of time calibration constants. */ +int64_t *calib_array; + +/** The size of the array of time calibration constants. */ +size_t calib_array_size; + +/** Hash table of task PIDs. */ +struct kshark_hash_id *tasks; + +/** A mutex, used to protect the access to the input file. */ +pthread_mutex_t input_mutex; + +/** Hash of tasks to filter on. */ +struct kshark_hash_id *show_task_filter; + +/** Hash of tasks to not display. */ +struct kshark_hash_id *hide_task_filter; + +/** Hash of events to filter on. */ +struct kshark_hash_id *show_event_filter; + +/** Hash of events to not display. */ +struct kshark_hash_id *hide_event_filter; + +/** Hash of CPUs to filter on. */ +struct kshark_hash_id *show_cpu_filter; + +/** Hash of CPUs to not display. */ +struct kshark_hash_id *hide_cpu_filter; + +/** List of Plugin interfaces. */ +struct kshark_dpi_list *plugins; + +/** The number of plugins registered for this stream.*/ +int n_plugins; + +/** The type of the data. */ +enum kshark_data_format format; + +/** List of Plugin's Event handlers. */ +struct kshark_event_proc_handler *event_handlers; + +/** List of Plugin's Draw handlers. */ +struct kshark_draw_handler *draw_handlers; + +/** +* The interface of methods used to operate over the data from a given +* stream. +*/ +struct kshark_data_stream_interface interface; +}; +/** Size of the task's hash table. */ #define KS_TASK_HASH_SHIFT 16#define KS_TASK_HASH_SIZE (1 << KS_TASK_HASH_SHIFT)
Signature · 1 line
2.25.1
PATCH 7/15 kernel-shark: Add stream_id to kshark_entry
In reply to Yordan Karadzhov (VMware)kshark_entry contains all information from one trace record, needed in order to visualize the time-series of trace records. Here we reorganize data fields to kshark_entry in order to make room for the unique identifier of the Data stream this entry belongs to.
--- src/libkshark.c | 2 +- src/libkshark.h | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-)
src/libkshark.c +1 -1
@@ -489,7 +489,7 @@ static inline void unset_event_filter_flag(struct kshark_context *kshark_ctx, e->visible &= ~event_mask; } -static void set_all_visible (uint16_t *v) { +static void set_all_visible (uint8_t *v) {/* Keep the original value of the PLUGIN_UNTOUCHED bit flag. */ *v |=0xFF & ~KS_PLUGIN_UNTOUCHED_MASK ; }
src/libkshark.h +9 -3
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: LGPL-2.1 */ /* -* Copyright (C) 2017 VMware Inc, Yordan Karadzhov <[email protected]> +* Copyright (C) 2017 VMware Inc, Yordan Karadzhov (VMware) <[email protected]> */ /**
@@ -50,7 +50,10 @@ struct kshark_entry {* kshark_filter_masks to check the level of visibility/invisibility * of the entry. */ -uint16_t visible; +uint8_t visible; + +/** Data stream identifier. */ +int8_t stream_id;/** The CPU core of the record. */ int16_t cpu;
@@ -58,7 +61,7 @@ struct kshark_entry {/** The PID of the task the record was generated. */ int32_t pid; -/** Unique Id ot the trace event type. */ +/** Unique Id of the trace event type. */ int32_t event_id;/** The offset into the trace file, used to find the record. */
@@ -351,6 +354,9 @@ struct kshark_data_stream {struct kshark_data_stream_interface interface; }; +/** Hard-coded maximum number of data stream. */ +#define KS_MAX_NUM_STREAMS 127 +/** Size of the task's hash table. */ #define KS_TASK_HASH_SHIFT 16#define KS_TASK_HASH_SIZE (1 << KS_TASK_HASH_SHIFT)
Signature · 1 line
2.25.1
PATCH 8/15 kernel-shark: Integrate the stream definitions with the C API
In reply to Yordan Karadzhov (VMware)Unfortunately this integration can only happen in a single change, which results into one huge patch, that may be hard to review. In this patch we modify the API itself plus the two most basic examples demonstrating how to use it. The most noticeable change that this new version of the API brings is that the processing of the trace-cmd/Ftrace data is wrapped inside the dedicated implementation of the Data stream interface and no trace-cmd definitions are leaking outside of src/libkshark-tepdata.c.
--- examples/datafilter.c | 67 +- examples/dataload.c | 22 +- src/CMakeLists.txt | 3 +- src/libkshark-collection.c | 121 ++- src/libkshark-plugin.c | 597 +++++++++-- src/libkshark-plugin.h | 279 ++++-- src/libkshark-tepdata.c | 1754 ++++++++++++++++++++++++++++++++ src/libkshark-tepdata.h | 100 ++ src/libkshark.c | 1928 +++++++++++++----------------------- src/libkshark.h | 414 +++++--- 10 files changed, 3699 insertions(+), 1586 deletions(-) create mode 100644 src/libkshark-tepdata.c create mode 100644 src/libkshark-tepdata.h
examples/datafilter.c +34 -33
@@ -7,22 +7,22 @@// C #include <stdio.h> #include <stdlib.h> +#include <string.h> // KernelShark #include "libkshark.h" +#include "libkshark-tepdata.h" const char *default_file ="trace.dat" ;int main (int argc,char **argv) { -ssize_t i, n_rows, n_tasks, n_evts, count; +size_t i, sd, n_rows, n_tasks, n_evts, count;struct kshark_context *kshark_ctx; +struct kshark_data_stream *stream;struct kshark_entry **data =NULL ; -struct tep_event_filter *adv_filter; -struct tep_event *event; +int *pids, *evt_ids;char *entry_str; -bool status; -int *pids;/* Create a new kshark session. */ kshark_ctx =NULL ;
@@ -31,32 +31,30 @@ int main(int argc, char **argv)/* Open a trace data file produced by trace-cmd. */ if (argc >1 ) - status =kshark_open (kshark_ctx, argv[1 ]); + sd =kshark_open (kshark_ctx, argv[1 ]);else - status =kshark_open (kshark_ctx, default_file); + sd =kshark_open (kshark_ctx, default_file); -if (!status) { +if (sd <0 ) {kshark_free (kshark_ctx);return 1 ; }/* Load the content of the file into an array of entries. */ - n_rows =kshark_load_data_entries (kshark_ctx, &data); -if (n_rows <1 ) { -kshark_free (kshark_ctx); -return 1 ; - } + n_rows =kshark_load_entries (kshark_ctx, sd, &data);/* Filter the trace data coming from trace-cmd. */ - n_tasks =kshark_get_task_pids (kshark_ctx, &pids); + n_tasks =kshark_get_task_pids (kshark_ctx, sd, &pids); + stream =kshark_get_data_stream (kshark_ctx, sd);for (i =0 ; i < n_tasks; ++i) { -const char *task_str = -tep_data_comm_from_pid (kshark_ctx->pevent, - pids[i]); +char *task_str = +kshark_comm_from_pid (sd, pids[i]);if (strcmp (task_str,"trace-cmd" ) ==0 ) -kshark_filter_add_id (kshark_ctx,KS_HIDE_TASK_FILTER , - pids[i]); +kshark_filter_add_id (kshark_ctx, sd, +KS_HIDE_TASK_FILTER , + pids[i]); +free (task_str); }free (pids);
@@ -66,7 +64,8 @@ int main(int argc, char **argv)* filterd entris in text format. */ kshark_ctx->filter_mask =KS_TEXT_VIEW_FILTER_MASK ; -kshark_filter_entries (kshark_ctx, data, n_rows); + kshark_ctx->filter_mask |=KS_EVENT_VIEW_FILTER_MASK ; +kshark_filter_stream_entries (kshark_ctx, sd, data, n_rows);/* Print to the screen the first 10 visible entries. */ count =0 ;
@@ -87,15 +86,19 @@ int main(int argc, char **argv)puts ("\n\n" );/* Show only "sched" events. */ - n_evts =tep_get_events_count (kshark_ctx->pevent); + n_evts = stream->n_events; + evt_ids =kshark_get_all_event_ids (kshark_ctx->stream[sd]);for (i =0 ; i < n_evts; ++i) { - event =tep_get_event (kshark_ctx->pevent, i); -if (strcmp (event->system,"sched" ) ==0 ) -kshark_filter_add_id (kshark_ctx,KS_SHOW_EVENT_FILTER , - event->id); +char *event_str = +kshark_event_from_id (sd, evt_ids[i]); +if (strstr (event_str,"sched/" )) +kshark_filter_add_id (kshark_ctx, sd, +KS_SHOW_EVENT_FILTER , + evt_ids[i]); +free (event_str); } -kshark_filter_entries (kshark_ctx, data, n_rows); +kshark_filter_stream_entries (kshark_ctx, sd, data, n_rows);/* Print to the screen the first 10 visible entries. */ count =0 ;
@@ -116,19 +119,17 @@ int main(int argc, char **argv)puts ("\n\n" );/* Clear all filters. */ -kshark_filter_clear (kshark_ctx,KS_HIDE_TASK_FILTER ); -kshark_filter_clear (kshark_ctx,KS_SHOW_EVENT_FILTER ); +kshark_filter_clear (kshark_ctx, sd,KS_HIDE_TASK_FILTER ); +kshark_filter_clear (kshark_ctx, sd,KS_SHOW_EVENT_FILTER );/* Use the Advanced filter to do event content based filtering. */ - adv_filter = kshark_ctx->advanced_event_filter; -tep_filter_add_filter_str (adv_filter, -"sched/sched_wakeup:target_cpu==1" ); +kshark_tep_add_filter_str (stream,"sched/sched_wakeup:target_cpu>1" );/* The Advanced filter requires reloading the data. */ for (i =0 ; i < n_rows; ++i)free (data[i]); - n_rows =kshark_load_data_entries (kshark_ctx, &data); + n_rows =kshark_load_entries (kshark_ctx, sd, &data); count =0 ;for (i =0 ; i < n_rows; ++i) {
@@ -149,7 +150,7 @@ int main(int argc, char **argv)free (data);/* Close the file. */ -kshark_close (kshark_ctx); +kshark_close (kshark_ctx, sd);/* Close the session. */ kshark_free (kshark_ctx);
examples/dataload.c +10 -12
@@ -19,8 +19,7 @@ int main(int argc, char **argv)struct kshark_entry **data =NULL ;ssize_t r, n_rows, n_tasks;char *entry_str; -bool status; -int *pids; +int sd, *pids;/* Create a new kshark session. */ kshark_ctx =NULL ;
@@ -29,30 +28,29 @@ int main(int argc, char **argv)/* Open a trace data file produced by trace-cmd. */ if (argc >1 ) - status =kshark_open (kshark_ctx, argv[1 ]); + sd =kshark_open (kshark_ctx, argv[1 ]);else - status =kshark_open (kshark_ctx, default_file); + sd =kshark_open (kshark_ctx, default_file); -if (!status) { +if (sd <0 ) {kshark_free (kshark_ctx);return 1 ; }/* Load the content of the file into an array of entries. */ - n_rows =kshark_load_data_entries (kshark_ctx, &data); + n_rows =kshark_load_entries (kshark_ctx, sd, &data);if (n_rows <1 ) {kshark_free (kshark_ctx);return 1 ; }/* Print to the screen the list of all tasks. */ - n_tasks =kshark_get_task_pids (kshark_ctx, &pids); + n_tasks =kshark_get_task_pids (kshark_ctx, sd, &pids);for (r =0 ; r < n_tasks; ++r) { -const char *task_str = -tep_data_comm_from_pid (kshark_ctx->pevent, - pids[r]); - +char *task_str = +kshark_comm_from_pid (sd, pids[r]);printf ("task: %s-%i\n" , task_str, pids[r]); +free (task_str); }free (pids);
@@ -82,7 +80,7 @@ int main(int argc, char **argv)free (data);/* Close the file. */ -kshark_close (kshark_ctx); +kshark_close (kshark_ctx, sd);/* Close the session. */ kshark_free (kshark_ctx);
src/CMakeLists.txt +2 -1
@@ -6,7 +6,8 @@ add_library(kshark SHARED libkshark.c # libkshark-model.c libkshark-plugin.c # libkshark-configio.c - libkshark-collection.c) + libkshark-collection.c + libkshark-tepdata.c) target_link_libraries(kshark ${TRACECMD_LIBRARY} ${TRACEFS_LIBRARY}
src/libkshark-collection.c +96 -25
268 lines not on this page · raw message
src/libkshark-plugin.c +504 -93
752 lines not on this page · raw message
src/libkshark-plugin.h +209 -70
355 lines not on this page · raw message
src/libkshark-tepdata.c new file +1754 -0
1,425 lines not on this page · raw message
src/libkshark-tepdata.h new file +100 -0
100 lines not on this page · raw message
src/libkshark.c +696 -1232
1,928 lines not on this page · raw message
src/libkshark.h +294 -120
414 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.25.1
Part of the diff is not shown here. View the raw message.
Re: [PATCH 08/15] kernel-shark: Integrate the stream definitions with the C API
I didn't apply the first patch, and nothing broke up until this patch.
Quoted 8 lines · On Tue, 29 Sep 2020 16:41:16 +0300 "Yordan Karadzhov (VMware)"…
Unfortunately this integration can only happen in a single change, which results into one huge patch, that may be hard to review. In this patch we modify the API itself plus the two most basic examples demonstrating how to use it. The most noticeable change that this new version of the API brings is that the processing of the trace-cmd/Ftrace data is wrapped inside the dedicated implementation of the Data stream interface and no trace-cmd definitions are leaking outside of
src/libkshark-tepdata.c.
I find it hard to believe that we can't break this up more.
Can you tell me what the old API did and what the new API does. I can probably help out in taking steps to go from one to the other. But I don't understand what exactly is changing without spending several days staring at this code.
-- Steve
Re: [PATCH 08/15] kernel-shark: Integrate the stream definitions with the C API
Hi Steven,
Quoted 15 lines · Steven Rostedt
I didn't apply the first patch, and nothing broke up until this patch.
Quoted 8 lines · On Tue, 29 Sep 2020 16:41:16 +0300 "Yordan Karadzhov (VMware)"…
Unfortunately this integration can only happen in a single change, which results into one huge patch, that may be hard to review. In this patch we modify the API itself plus the two most basic examples demonstrating how to use it. The most noticeable change that this new version of the API brings is that the processing of the trace-cmd/Ftrace data is wrapped inside the dedicated implementation of the Data stream interface and no trace-cmd definitions are leaking outside of
src/libkshark-tepdata.c.
Quoted 6 lines
I find it hard to believe that we can't break this up more.
Can you tell me what the old API did and what the new API does. I can probably help out in taking steps to go from one to the other. But I don't understand what exactly is changing without spending several days staring at this code.
The essence of the change is that in the API we are not allowed to include any trace-cmd header files. And because we have been using trace-cmd definitions and methods everywhere, once we remove the header inclusion everything breaks. The only place that is allowed to include trace-cmd stuff is the implementation of the readout interface for trace-cmd data (src/libkshark-tepdata.c).
I understand that the approach I am taking with the introduction of the regression is against all rules and is a BIG compromise. This is the reason why I wanted to keep at least the two very basic example applications functional during the change. Of course we can further break this huge and ugly change into smaller changes that have better logical connection, but in such a case it will be very very hard to keep the examples functional during those changes.
Thanks!
Yordan
Quoted 3 lines
-- Steve
Re: [PATCH 08/15] kernel-shark: Integrate the stream definitions with the C API
In reply to Yordan Karadzhov (VMware)Quoted 33 lines · On Tue, 29 Sep 2020 16:41:16 +0300 "Yordan Karadzhov (VMware)"…
examples/datafilter.c +5
-5@@ -7,22 +7,22 @@// C #include <stdio.h> #include <stdlib.h> +#include <string.h> // KernelShark #include "libkshark.h" +#include "libkshark-tepdata.h" const char *default_file ="trace.dat" ;int main (int argc,char **argv) { -ssize_t i, n_rows, n_tasks, n_evts, count; +size_t i, sd, n_rows, n_tasks, n_evts, count;struct kshark_context *kshark_ctx; +struct kshark_data_stream *stream;struct kshark_entry **data =NULL ; -struct tep_event_filter *adv_filter; -struct tep_event *event; +int *pids, *evt_ids;char *entry_str; -bool status; -int *pids;/* Create a new kshark session. */ kshark_ctx =NULL ;
examples/datafilter.c int main(int argc, char **argv) · line 36
Quoted 8 lines
examples/datafilter.c +2
-2@@ -31,32 +31,30 @@ int main(int argc, char **argv)/* Open a trace data file produced by trace-cmd. */ if (argc >1 ) - status =kshark_open (kshark_ctx, argv[1 ]); + sd =kshark_open (kshark_ctx, argv[1 ]);else - status =kshark_open (kshark_ctx, default_file); + sd =kshark_open (kshark_ctx, default_file);
OK, here's a perfect example of how I would start breaking this up.
I would implement the switch to using this integer, and add it to all the current old APIs (don't worry about adding new ones with this change).
Quoted 140 lines
examples/datafilter.c +1
-6-if (!status) { +if (sd <0 ) {kshark_free (kshark_ctx);return 1 ; }/* Load the content of the file into an array of entries. */ - n_rows =kshark_load_data_entries (kshark_ctx, &data); -if (n_rows <1 ) { -kshark_free (kshark_ctx); -return 1 ; - }src/libkshark.c +14
-54@@ -1,26 +1,28 @@// SPDX-License-Identifier: LGPL-2.1 /* -* Copyright (C) 2017 VMware Inc, Yordan Karadzhov <[email protected]> +* Copyright (C) 2017 VMware Inc, Yordan Karadzhov (VMware) <[email protected]> */ /** * @file libkshark.c -* @brief API for processing of FTRACE (trace-cmd) data. +* @brief API for processing of traceing data. */ +#ifndef _GNU_SOURCE/** Use GNU C Library. */ -#define _GNU_SOURCE 1 +#define _GNU_SOURCE +#endif // _GNU_SOURCE // C #include <stdlib.h> #include <stdio.h> #include <assert.h> +#include <string.h> // KernelShark #include "libkshark.h" - -static __threadstruct trace_seq seq; +#include "libkshark-tepdata.h" static struct kshark_context *kshark_context_handler =NULL ;@@ -32,18 +34,11 @@ static bool kshark_default_context(struct kshark_context **context)if (!kshark_ctx)return false; - kshark_ctx->event_handlers =NULL ; - kshark_ctx->collections =NULL ; - kshark_ctx->plugins =NULL ; - - kshark_ctx->show_task_filter =tracecmd_filter_id_hash_alloc (); - kshark_ctx->hide_task_filter =tracecmd_filter_id_hash_alloc (); - - kshark_ctx->show_event_filter =tracecmd_filter_id_hash_alloc (); - kshark_ctx->hide_event_filter =tracecmd_filter_id_hash_alloc (); + kshark_ctx->stream =calloc (KS_MAX_NUM_STREAMS , +sizeof (*kshark_ctx->stream)); - kshark_ctx->show_cpu_filter =tracecmd_filter_id_hash_alloc (); - kshark_ctx->hide_cpu_filter =tracecmd_filter_id_hash_alloc (); + kshark_ctx->collections =NULL ; + kshark_ctx->inputs =NULL ; kshark_ctx->filter_mask =0x0 ;@@ -58,14 +53,6 @@ static bool kshark_default_context(struct kshark_context **context)return true; } -static bool init_thread_seq (void ) -{ -if (!seq.buffer) -trace_seq_init (&seq); - -return seq.buffer !=NULL ; -} -/** * @brief Initialize a kshark session. This function must be called before * calling any other kshark function. If the session has been @@ -102,1466 +89,809 @@ bool kshark_instance(struct kshark_context **kshark_ctx) } } -if (!init_thread_seq ()) -return false; -return true; } -static void kshark_free_task_list (struct kshark_context *kshark_ctx) -{ -struct kshark_task_list *task; -int i; - -if (!kshark_ctx) -return ; - -for (i =0 ; i <KS_TASK_HASH_SIZE ; ++i) { -while (kshark_ctx->tasks[i]) { - task = kshark_ctx->tasks[i]; - kshark_ctx->tasks[i] = task->next; -free (task); - } - } -} -/** * @brief Open and prepare for reading a trace data file specified by "file". -* If the specified file does not exist, or contains no trace data, -* the function returns false. * * @param kshark_ctx: Input location for context pointer. * @param file: The file to load. * -* @returns True on success, or false on failure. +* @returns The Id number of the data stream associated with this file on success. +* Otherwise a negative errno code. */ -bool kshark_open (struct kshark_context *kshark_ctx,const char *file) +int kshark_open (struct kshark_context *kshark_ctx,const char *file) { -struct tracecmd_input *handle; - -kshark_free_task_list (kshark_ctx); - - handle =tracecmd_open (file); -if (!handle)
src/libkshark.c bool kshark_instance(struct kshark_context **kshark_ctx) · line 110
Quoted 10 lines
src/libkshark.c +4
-5-return false; +int sd, rt; -if (pthread_mutex_init (&kshark_ctx->input_mutex,NULL ) !=0 ) { -tracecmd_close (handle); -return false; - } + sd =kshark_add_stream (kshark_ctx); +if (sd <0 ) +return sd;
So what this can do is add the stream, but have the kshark_data_stream have a tracecmd_handle in its field. And then you can still have:
stream->handle = tracecmd_open(...);
This way, you can pass around the stream index in this step, but everything still requiring to use the handle.
Then the next step would be to start abstracting each operation a little more, until everything can use the stream directly. And then you remove the handle.
This is what I mean by slowly moving to a new API. The in between steps may not look like either API. But that's OK.
And it makes it much easier to review. Because honestly, this current approach isn't much better than just squashing this all into one patch, and hoping it works.
-- Steve
PATCH 9/15 kernel-shark: Provide merging of multiple data streams
In reply to Yordan Karadzhov (VMware)The C API provides loading of the trace data in two different forms. The firs one is an array of kshark_entries and is being used by the KernelShark GUI. The second is a matrix-like structure that has all the fields of the kshark_entry stored in separate arrays, forming the columns of the matrix. The second form of the data is used by trace-cruncher. In this patch we add methods for merging of several data streams into a single data set. Both kshark_entries and matrix forms of the data are supported. This patch includes a simple example that demonstrate how to open a file that contains multiple buffers. Each buffers is loaded into a separate Data stream and those streams are merged together.
--- examples/CMakeLists.txt | 4 + examples/multibufferload.c | 60 +++++++++ src/libkshark.c | 255 +++++++++++++++++++++++++++++++++++++ src/libkshark.h | 47 +++++++ 4 files changed, 366 insertions(+) create mode 100644 examples/multibufferload.c
examples/CMakeLists.txt +4 -0
@@ -4,6 +4,10 @@ message(STATUS "dataload") add_executable(dload dataload.c) target_link_libraries(dload kshark) +message(STATUS "multibufferload") +add_executable(mbload multibufferload.c) +target_link_libraries(mbload kshark) + message(STATUS "datafilter") add_executable(dfilter datafilter.c) target_link_libraries(dfilter kshark)
examples/multibufferload.c new file +60 -0
60 lines not on this page · raw message
src/libkshark.c +255 -0
264 lines not on this page · raw message
src/libkshark.h +47 -0
59 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH 10/15 kernel-shark: Integrate the stream definitions with data model
In reply to Yordan Karadzhov (VMware)The visualization model is adapted in order to be able to distinguish the entries belonging to different Data streams.
We re-enable the visualization model example as well.
--- examples/CMakeLists.txt | 8 ++-- examples/datahisto.c | 38 +++++++-------- src/CMakeLists.txt | 2 +- src/libkshark-model.c | 102 ++++++++++++++++++++++++---------------- src/libkshark-model.h | 34 +++++++------- 5 files changed, 102 insertions(+), 82 deletions(-)
examples/CMakeLists.txt +4 -4
@@ -12,10 +12,10 @@ message(STATUS "datafilter") add_executable(dfilter datafilter.c) target_link_libraries(dfilter kshark) -# message(STATUS "datahisto") -# add_executable(dhisto datahisto.c) -# target_link_libraries(dhisto kshark) -# +message(STATUS "datahisto") +add_executable(dhisto datahisto.c) +target_link_libraries(dhisto kshark) + # message(STATUS "confogio") # add_executable(confio configio.c) # target_link_libraries(confio kshark)
examples/datahisto.c +19 -19
@@ -7,6 +7,7 @@// C #include <stdio.h> #include <stdlib.h> +#include <string.h> // KernelShark #include "libkshark.h"
@@ -16,7 +17,7 @@const char *default_file ="trace.dat" ; -void dump_bin (struct kshark_trace_histo *histo,int bin, +void dump_bin (struct kshark_trace_histo *histo,int bin,int sd,const char *type,int val) {const struct kshark_entry *e_front, *e_back;
@@ -26,22 +27,22 @@ void dump_bin(struct kshark_trace_histo *histo, int bin,printf ("bin %i {\n" , bin);if (strcmp (type,"cpu" ) ==0 ) { e_front =ksmodel_get_entry_front (histo, bin, true, - kshark_match_cpu, val, + kshark_match_cpu, sd, &val,NULL , &i_front); e_back =ksmodel_get_entry_back (histo, bin, true, - kshark_match_cpu, val, + kshark_match_cpu, sd, &val,NULL , &i_back); }else if (strcmp (type,"task" ) ==0 ) { e_front =ksmodel_get_entry_front (histo, bin, true, - kshark_match_pid, val, + kshark_match_pid, sd, &val,NULL , &i_front); e_back =ksmodel_get_entry_back (histo, bin, true, - kshark_match_pid, val, + kshark_match_pid, sd, &val,NULL , &i_back); }else {
@@ -67,12 +68,12 @@ void dump_bin(struct kshark_trace_histo *histo, int bin,puts ("}\n" ); } -void dump_histo (struct kshark_trace_histo *histo,const char *type,int val) +void dump_histo (struct kshark_trace_histo *histo,int sd,const char *type,int val) {size_t bin;for (bin =0 ; bin < histo->n_bins; ++bin) -dump_bin (histo, bin, type, val); +dump_bin (histo, bin, sd, type, val); }int main (int argc,char **argv)
@@ -81,8 +82,7 @@ int main(int argc, char **argv)struct kshark_entry **data =NULL ;struct kshark_trace_histo histo;ssize_t i, n_rows, n_tasks; -bool status; -int *pids; +int sd, *pids;/* Create a new kshark session. */ kshark_ctx =NULL ;
@@ -91,24 +91,24 @@ int main(int argc, char **argv)/* Open a trace data file produced by trace-cmd. */ if (argc >1 ) - status =kshark_open (kshark_ctx, argv[1 ]); + sd =kshark_open (kshark_ctx, argv[1 ]);else - status =kshark_open (kshark_ctx, default_file); + sd =kshark_open (kshark_ctx, default_file); -if (!status) { +if (sd <0 ) {kshark_free (kshark_ctx);return 1 ; }/* Load the content of the file into an array of entries. */ - n_rows =kshark_load_data_entries (kshark_ctx, &data); + n_rows =kshark_load_entries (kshark_ctx, sd, &data);if (n_rows <1 ) {kshark_free (kshark_ctx);return 1 ; }/* Get a list of all tasks. */ - n_tasks =kshark_get_task_pids (kshark_ctx, &pids); + n_tasks =kshark_get_task_pids (kshark_ctx, sd, &pids);/* Initialize the Visualization Model. */ ksmodel_init (&histo);
@@ -119,7 +119,7 @@ int main(int argc, char **argv)ksmodel_fill (&histo, data, n_rows);/* Dump the raw bins. */ -dump_histo (&histo,"" ,0 ); +dump_histo (&histo, sd,"" ,0 );puts ("\n...\n\n" );
@@ -127,13 +127,13 @@ int main(int argc, char **argv)* Change the state of the model. Do 50% Zoom-In and dump only CPU 0. */ ksmodel_zoom_in (&histo,.50 ,-1 ); -dump_histo (&histo,"cpu" ,0 ); +dump_histo (&histo, sd,"cpu" ,0 );puts ("\n...\n\n" );/* Shift forward by two bins and this time dump only CPU 1. */ ksmodel_shift_forward (&histo,2 ); -dump_histo (&histo,"cpu" ,1 ); +dump_histo (&histo, sd,"cpu" ,1 );puts ("\n...\n\n" );
@@ -142,7 +142,7 @@ int main(int argc, char **argv)* Task. */ ksmodel_zoom_out (&histo,.10 ,N_BINS -1 ); -dump_histo (&histo,"task" , pids[n_tasks -1 ]); +dump_histo (&histo, sd,"task" , pids[n_tasks -1 ]);/* Reset (clear) the model. */ ksmodel_clear (&histo);
@@ -154,7 +154,7 @@ int main(int argc, char **argv)free (data);/* Close the file. */ -kshark_close (kshark_ctx); +kshark_close (kshark_ctx, sd);/* Close the session. */ kshark_free (kshark_ctx);
src/CMakeLists.txt +1 -1
@@ -3,7 +3,7 @@ message("\n src ...") message(STATUS "libkshark") add_library(kshark SHARED libkshark.c libkshark-hash.c -# libkshark-model.c + libkshark-model.c libkshark-plugin.c # libkshark-configio.c libkshark-collection.c
src/libkshark-model.c +61 -41
358 lines not on this page · raw message
src/libkshark-model.h +17 -17
@@ -1,13 +1,13 @@/* SPDX-License-Identifier: LGPL-2.1 */ /* -* Copyright (C) 2017 VMware Inc, Yordan Karadzhov <[email protected]> +* Copyright (C) 2017 VMware Inc, Yordan Karadzhov (VMware) <[email protected]> */ -/** -* @file libkshark-model.h -* @brief Visualization model for FTRACE (trace-cmd) data. -*/ +/** +* @file libkshark-model.h +* @brief Time series visualization model for tracing data. +*/ #ifndef _LIB_KSHARK_MODEL_H#define _LIB_KSHARK_MODEL_H
@@ -102,64 +102,64 @@ ssize_t ksmodel_first_index_at_bin(struct kshark_trace_histo *histo, int bin);ssize_t ksmodel_last_index_at_bin (struct kshark_trace_histo *histo,int bin);ssize_t ksmodel_first_index_at_cpu (struct kshark_trace_histo *histo, -int bin,int cpu); +int bin,int sd,int cpu);ssize_t ksmodel_first_index_at_pid (struct kshark_trace_histo *histo, -int bin,int pid); +int bin,int sd,int pid);const struct kshark_entry *ksmodel_get_entry_front (struct kshark_trace_histo *histo,int bin,bool vis_only, -matching_condition_func func,int val, +matching_condition_func func,int sd,int *values,struct kshark_entry_collection *col,ssize_t *index);const struct kshark_entry *ksmodel_get_entry_back (struct kshark_trace_histo *histo,int bin,bool vis_only, -matching_condition_func func,int val, +matching_condition_func func,int sd,int *values,struct kshark_entry_collection *col,ssize_t *index);int ksmodel_get_pid_front (struct kshark_trace_histo *histo, -int bin,int cpu,bool vis_only, +int bin,int sd,int cpu,bool vis_only,struct kshark_entry_collection *col,ssize_t *index);int ksmodel_get_pid_back (struct kshark_trace_histo *histo, -int bin,int cpu,bool vis_only, +int bin,int sd,int cpu,bool vis_only,struct kshark_entry_collection *col,ssize_t *index);int ksmodel_get_cpu_front (struct kshark_trace_histo *histo, -int bin,int pid,bool vis_only, +int bin,int sd,int pid,bool vis_only,struct kshark_entry_collection *col,ssize_t *index);int ksmodel_get_cpu_back (struct kshark_trace_histo *histo, -int bin,int pid,bool vis_only, +int bin,int sd,int pid,bool vis_only,struct kshark_entry_collection *col,ssize_t *index);bool ksmodel_cpu_visible_event_exist (struct kshark_trace_histo *histo, -int bin,int cpu, +int bin,int sd,int cpu,struct kshark_entry_collection *col,ssize_t *index);bool ksmodel_task_visible_event_exist (struct kshark_trace_histo *histo, -int bin,int pid, +int bin,int sd,int pid,struct kshark_entry_collection *col,ssize_t *index);const struct kshark_entry *ksmodel_get_cpu_missed_events (struct kshark_trace_histo *histo, -int bin,int cpu, +int bin,int sd,int cpu,struct kshark_entry_collection *col,ssize_t *index);const struct kshark_entry *ksmodel_get_task_missed_events (struct kshark_trace_histo *histo, -int bin,int pid, +int bin,int sd,int pid,struct kshark_entry_collection *col,ssize_t *index);
Signature · 1 line
2.25.1
PATCH 11/15 kernel-shark: Use only signed types for model defs
In reply to Yordan Karadzhov (VMware)All variables that are used to describe the state of the model, like the range of the model or the size of the bin must have signed integer types. This is a continuation of the change started by switching to signed timestamps.
--- src/libkshark-model.c | 12 ++++++------ src/libkshark-model.h | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-)
src/libkshark-model.c +6 -6
@@ -91,10 +91,10 @@ static bool ksmodel_histo_alloc(struct kshark_trace_histo *histo, size_t n) }static void ksmodel_set_in_range_bining (struct kshark_trace_histo *histo, -size_t n,uint64_t min,uint64_t max, +size_t n,int64_t min,int64_t max,bool force_in_range) { -uint64_t corrected_range, delta_range, range = max - min; +int64_t corrected_range, delta_range, range = max - min;struct kshark_entry *last;/* The size of the bin must be >= 1, hence the range must be >= n. */
@@ -163,7 +163,7 @@ static void ksmodel_set_in_range_bining(struct kshark_trace_histo *histo,* @param max: Upper edge of the time-window to be visualized. */ void ksmodel_set_bining (struct kshark_trace_histo *histo, -size_t n,uint64_t min,uint64_t max) +size_t n,int64_t min,int64_t max) {ksmodel_set_in_range_bining (histo, n, min, max, false); }
@@ -256,7 +256,7 @@ static size_t ksmodel_set_upper_edge(struct kshark_trace_histo *histo)static void ksmodel_set_next_bin_edge (struct kshark_trace_histo *histo,size_t bin,size_t last_row) { -uint64_t time_min, time_max; +int64_t time_min, time_max;size_t next_bin = bin +1 ;ssize_t row;
@@ -598,9 +598,9 @@ void ksmodel_shift_backward(struct kshark_trace_histo *histo, size_t n)* @param histo: Input location for the model descriptor. * @param ts: position in time to be visualized. */ -void ksmodel_jump_to (struct kshark_trace_histo *histo,uint64_t ts) +void ksmodel_jump_to (struct kshark_trace_histo *histo,int64_t ts) { -uint64_t min, max, range_min; +int64_t min, max, range_min;if (ts > histo->min && ts < histo->max) {/*
src/libkshark-model.h +9 -9
@@ -58,16 +58,16 @@ struct kshark_trace_histo {* Lower edge of the time-window to be visualized. Only entries having * timestamp >= min will be visualized. */ -uint64_t min; +int64_t min;/** * Upper edge of the time-window to be visualized. Only entries having * timestamp <= max will be visualized. */ -uint64_t max; +int64_t max;/** The size in time for each bin. */ -uint64_t bin_size; +int64_t bin_size;/** Number of bins. */ int n_bins;
@@ -78,7 +78,7 @@ void ksmodel_init(struct kshark_trace_histo *histo);void ksmodel_clear (struct kshark_trace_histo *histo);void ksmodel_set_bining (struct kshark_trace_histo *histo, -size_t n,uint64_t min,uint64_t max); +size_t n,int64_t min,int64_t max);void ksmodel_fill (struct kshark_trace_histo *histo,struct kshark_entry **data,size_t n);
@@ -89,7 +89,7 @@ void ksmodel_shift_forward(struct kshark_trace_histo *histo, size_t n);void ksmodel_shift_backward (struct kshark_trace_histo *histo,size_t n); -void ksmodel_jump_to (struct kshark_trace_histo *histo,uint64_t ts); +void ksmodel_jump_to (struct kshark_trace_histo *histo,int64_t ts);void ksmodel_zoom_out (struct kshark_trace_histo *histo,double r,int mark);
@@ -163,16 +163,16 @@ ksmodel_get_task_missed_events(struct kshark_trace_histo *histo,struct kshark_entry_collection *col, ssize_t *index); -static inline double ksmodel_bin_time (struct kshark_trace_histo *histo, +static inline int64_t ksmodel_bin_ts (struct kshark_trace_histo *histo,int bin) { -return (histo->min + bin*histo->bin_size) *1e-9 ; +return (histo->min + bin*histo->bin_size); } -static inline uint64_t ksmodel_bin_ts (struct kshark_trace_histo *histo, +static inline double ksmodel_bin_time (struct kshark_trace_histo *histo,int bin) { -return (histo->min + bin*histo->bin_size); +return ksmodel_bin_ts (histo, bin) *1e-9 ; }#ifdef __cplusplus
Signature · 1 line
2.25.1
PATCH 12/15 kernel-shark: Add ksmodel_get_bin()
In reply to Yordan Karadzhov (VMware)The method retrieves the Id of the bin of the model that contains given entry.
--- src/libkshark-model.c | 27 +++++++++++++++++++++++++++ src/libkshark-model.h | 3 +++ 2 files changed, 30 insertions(+)
src/libkshark-model.c +27 -0
@@ -1325,3 +1325,30 @@ ksmodel_get_task_missed_events(struct kshark_trace_histo *histo, match_pid_missed_events, sd, &pid, col, index); } + +/** +* @brief Find the bin Id of a give entry. +* +* @param histo: Input location for the model descriptor. +* @param entry: Input location for entry. +* +* @returns If the timestamp of the entry is inside the range of the model the +* function returns the Id of the bin it belongs to. +* If the timestamp of the entry is outside of the range of the model +* the function returns UPPER_OVERFLOW_BIN or LOWER_OVERFLOW_BIN +* (negative values). +*/ +int ksmodel_get_bin (struct kshark_trace_histo *histo, +const struct kshark_entry *entry) +{ +if (entry->ts < histo->min) +return UPPER_OVERFLOW_BIN ; + +if (entry->ts > histo->max) +return LOWER_OVERFLOW_BIN ; + +if (entry->ts == histo->max) +return histo->n_bins -1 ; + +return (entry->ts - histo->min) / histo->bin_size; +}
src/libkshark-model.h +3 -0
@@ -175,6 +175,9 @@ static inline double ksmodel_bin_time(struct kshark_trace_histo *histo,return ksmodel_bin_ts (histo, bin) *1e-9 ; } +int ksmodel_get_bin (struct kshark_trace_histo *histo, +const struct kshark_entry *entry); + #ifdef __cplusplus }#endif // __cplusplus
Signature · 1 line
2.25.1
PATCH 13/15 kernel-shark: Protect ksmodel_set_in_range_bining()
In reply to Yordan Karadzhov (VMware)Handle the case when the number of bins is zero or negative.
--- src/libkshark-model.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-)
src/libkshark-model.c +12 -1
@@ -97,8 +97,19 @@ static void ksmodel_set_in_range_bining(struct kshark_trace_histo *histo,int64_t corrected_range, delta_range, range = max - min;struct kshark_entry *last; +if (n <=0 ) { + histo->n_bins = histo->bin_size =0 ; + histo->min = min; + histo->max = max; + +free (histo->bin_count); +free (histo->map); + +return ; + } +/* The size of the bin must be >= 1, hence the range must be >= n. */ -if (n ==0 || range < n) { +if (range < n) { range = n; max = min + n; }
Signature · 1 line
2.25.1
PATCH 14/15 kernel-shark: Add methods for time calibration
In reply to Yordan Karadzhov (VMware)We add an infrastructure for correcting the timestamps of the entries. This is needed in order to correlate Data streams that have been recorded using non-synchronized clocks. The infrastructure can handle an arbitrary timestamps correction formula, however for the moment we only provide calibration that adds a constant offset.
--- src/libkshark.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ src/libkshark.h | 6 +++++ 2 files changed, 73 insertions(+)
src/libkshark.c +67 -0
@@ -1184,6 +1184,73 @@ kshark_get_entry_back(const struct kshark_entry_request *req,return get_entry (req, data, index, req->first, end,-1 ); } +static int compare_time (const void * a,const void * b) +{ +const struct kshark_entry *entry_a, *entry_b; + + entry_a = *(const struct kshark_entry **) a; + entry_b = *(const struct kshark_entry **) b; + +if (entry_a->ts > entry_b->ts) +return 1 ; + +if (entry_a->ts < entry_b->ts) +return -1 ; + +return 0 ; +} + +static void kshark_data_qsort (struct kshark_entry **entries,size_t size) +{ +qsort (entries, size,sizeof (struct kshark_entry *), compare_time); +} + +/** +* Add constant offset to the timestamp of the entry. To be used by the sream +* object as a System clock calibration callback function. +*/ +void kshark_offset_calib (int64_t *ts,int64_t *argv) +{ + *ts += argv[0 ]; +} + +/** +* @brief Apply constant offset to the timestamps of all entries from a given +* Data stream. +* +* @param kshark_ctx: Input location for the session context pointer. +* @param entries: Input location for the trace data. +* @param size: The size of the trace data. +* @param sd: Data stream identifier. +* @param offset: The constant offset to be added (in nanosecond). +*/ +void kshark_set_clock_offset (struct kshark_context *kshark_ctx, +struct kshark_entry **entries,size_t size, +int sd,int64_t offset) +{ +struct kshark_data_stream *stream; +int64_t correction; + + stream =kshark_get_data_stream (kshark_ctx, sd); +if (!stream) +return ; + +if (!stream->calib_array) { + stream->calib = kshark_offset_calib; + stream->calib_array =calloc (1 ,sizeof (*stream->calib_array)); + stream->calib_array_size =1 ; + } + + correction = offset - stream->calib_array[0 ]; + stream->calib_array[0 ] = offset; + +for (size_t i =0 ; i < size; ++i) +if (entries[i]->stream_id == sd) + entries[i]->ts += correction; + +kshark_data_qsort (entries, size); +} +static int first_in_time_entry (struct kshark_entry_data_set *buffer,int n_buffers,size_t *count) {int64_t t_min =INT64_MAX ;
src/libkshark.h +6 -0
@@ -1096,6 +1096,12 @@ struct kshark_config_doc *kshark_open_config_file(const char *file_name,struct kshark_config_doc *kshark_json_to_conf (struct json_object *jobj); +void kshark_offset_calib (int64_t *ts,int64_t *atgv); + +void kshark_set_clock_offset (struct kshark_context *kshark_ctx, +struct kshark_entry **entries,size_t size, +int sd,int64_t offset); +/** Structure representing a data set made of KernelShark entries. */ struct kshark_entry_data_set {/** Array of entries pointers. */
Signature · 1 line
2.25.1
PATCH 15/15 kernel-shark: Integrate streams with libkshark-configio
In reply to Yordan Karadzhov (VMware)The configuration input/output is adapted in order to be able to work with the new version of the C API. Now it can handle multiple Data streams. We are re-enabling the corresponding example as well.
--- examples/CMakeLists.txt | 8 +- examples/configio.c | 20 +- src/CMakeLists.txt | 2 +- src/libkshark-configio.c | 1175 ++++++++++++++++++++++++++++++++------ src/libkshark.h | 73 ++- 5 files changed, 1062 insertions(+), 216 deletions(-)
examples/CMakeLists.txt +4 -4
@@ -16,10 +16,10 @@ message(STATUS "datahisto") add_executable(dhisto datahisto.c) target_link_libraries(dhisto kshark) -# message(STATUS "confogio") -# add_executable(confio configio.c) -# target_link_libraries(confio kshark) -# +message(STATUS "confogio") +add_executable(confio configio.c) +target_link_libraries(confio kshark) + # message(STATUS "dataplot") # add_executable(dplot dataplot.cpp) # target_link_libraries(dplot kshark-plot)
examples/configio.c +12 -8
@@ -7,22 +7,26 @@ int main(int argc, char **argv) {struct kshark_config_doc *conf, *filter, *hello;struct kshark_context *kshark_ctx; -int *ids =NULL , i; +struct kshark_data_stream *stream; +int sd, *ids =NULL , i;/* Create a new kshark session. */ kshark_ctx =NULL ;if (!kshark_instance (&kshark_ctx))return 1 ; + sd =kshark_add_stream (kshark_ctx); + stream = kshark_ctx->stream[sd]; +if (argc ==1 ) { -tracecmd_filter_id_add (kshark_ctx->show_task_filter,314 ); -tracecmd_filter_id_add (kshark_ctx->show_task_filter,42 ); +kshark_hash_id_add (stream->show_task_filter,314 ); +kshark_hash_id_add (stream->show_task_filter,42 );/* Create a new Confog. doc. */ conf =kshark_config_new ("foo.bar.config" ,KS_CONFIG_JSON );/* Add filter's info. */ - filter =kshark_export_all_filters (kshark_ctx,KS_CONFIG_JSON ); + filter =kshark_export_all_filters (kshark_ctx, sd,KS_CONFIG_JSON );kshark_config_doc_add (conf,"Filters" ,filter);/* Add "Hello Kernel" message. */
@@ -39,11 +43,11 @@ int main(int argc, char **argv)/* Retrieve the filter's info. */ filter =kshark_config_alloc (KS_CONFIG_JSON );if (kshark_config_doc_get (conf,"Filters" ,filter)) { -kshark_import_all_filters (kshark_ctx, filter); +kshark_import_all_filters (kshark_ctx, sd, filter);/* Get the array of Ids to be fitered. */ - ids =tracecmd_filter_ids (kshark_ctx->show_task_filter); -for (i =0 ; i < kshark_ctx->show_task_filter->count; ++i) + ids =kshark_hash_ids (stream->show_task_filter); +for (i =0 ; i < stream->show_task_filter->count; ++i)printf ("pid: %i\n" , ids[i]); }
@@ -58,7 +62,7 @@ int main(int argc, char **argv) }kshark_free_config_doc (conf); - +kshark_close (kshark_ctx, sd);kshark_free (kshark_ctx);return 0 ;
src/CMakeLists.txt +1 -1
@@ -5,7 +5,7 @@ add_library(kshark SHARED libkshark.c libkshark-hash.c libkshark-model.c libkshark-plugin.c -# libkshark-configio.c + libkshark-configio.c libkshark-collection.c libkshark-tepdata.c)
src/libkshark-configio.c +986 -189
1,726 lines not on this page · raw message
src/libkshark.h +59 -14
129 lines not on this page · raw message
Signature · 1 line
2.25.1