Skip to contentSkip to conversation

kernel-shark: Visualization plugin tools

PATCH New Yordan Karadzhov (VMware) 1 participant 7 messages Jan 8, 2021

lore
Share & tools
lore
#0 lore raw

v3 cover kernel-shark: Visualization plugin tools

We implement a general purpose instruments, that can be used to implement efficient (fast) visualization plugins. We also use the new instrumentation to optimize the sched_events plugin.

Changes in v3:
* Typo fixes in [patch 2]
* The missing signed-off-by of [patch 3] is restored.

Changes in v2:
* KS_DOUBLE_SIZE macro was simplified, based on the suggestion made by Steven in his review. A dedicated test case for the macro has been added.
* Bug fix in kshark_free_data_container().
* Bug fix in kshark_data_container_append().

Yordan Karadzhov (VMware) (6):
kernel-shark: Add KS_DOUBLE_SIZE macro
kernel-shark: Add kshark_data_container to libkshark
kernel-shark: Add KS_DEFINE_PLUGIN_CONTEXT macro
kernel-shark: Start using C++17
kernel-shark: Add plotting methods to KsPlugins
kernel-shark: Speed-up the sched_events plugin

 CMakeLists.txt              |   8 +-
 src/CMakeLists.txt          |   5 +-
 src/KsPlugins.cpp           | 416 ++++++++++++++++++++++++++++++++++++
 src/KsPlugins.hpp           |  48 +++++
 src/libkshark-plugin.h      |  59 +++++
 src/libkshark-tepdata.c     |  34 +++
 src/libkshark-tepdata.h     |   7 +
 src/libkshark.c             | 158 +++++++++++++-
 src/libkshark.h             |  43 ++++
 src/plugins/CMakeLists.txt  |  11 +-
 src/plugins/SchedEvents.cpp | 310 +++++++--------------------
 src/plugins/sched_events.c  | 393 +++++++++++-----------------------
 src/plugins/sched_events.h  |  50 ++---
 tests/libkshark-tests.cpp   |  89 ++++++++
 14 files changed, 1071 insertions(+), 560 deletions(-)
 create mode 100644 src/KsPlugins.cpp
Signature · 1 line
2.25.1
#1 lore raw

PATCH v3 1/6 kernel-shark: Add KS_DOUBLE_SIZE macro

The macro is useful for resizing of dynamic arrays. It is currently used to resize the Data stream descriptor array, owned by the session context. We will later use the macro with the arrays of data fields and plugin contexts.

---
 src/libkshark-plugin.h    | 17 +++++++++++++++++
 src/libkshark.c           | 11 ++---------
 tests/libkshark-tests.cpp | 20 ++++++++++++++++++++
 3 files changed, 39 insertions(+), 9 deletions(-)
src/libkshark-plugin.h +17 -0
index 1a642ad..4104357 100644
@@ -346,6 +346,23 @@ int kshark_handle_dpi(struct kshark_data_stream *stream,
 int kshark_handle_all_dpis(struct kshark_data_stream *stream,
 			   enum kshark_plugin_actions  task_id);
 
+/** General purpose macro for resizing dynamic arrays. */
+#define KS_DOUBLE_SIZE(array, size)					\
+({									\
+	ssize_t __n = size;						\
+	bool __ok = false;						\
+	__typeof__(array) __tmp =					\
+		(__typeof__(array)) realloc(array,			\
+					    2 * __n * sizeof(*__tmp));	\
+	if (__tmp) {							\
+		memset(__tmp + __n, 0, __n * sizeof(*__tmp));		\
+		size = 2 * __n;						\
+		array = __tmp;						\
+		__ok = true;						\
+	}								\
+	__ok;								\
+})									\
+
 #ifdef __cplusplus
 }
 #endif // __cplusplus
src/libkshark.c +2 -9
index 315c24f..0e2ce7a 100644
@@ -234,16 +234,9 @@ int kshark_add_stream(struct kshark_context *kshark_ctx)
 
 	if (kshark_ctx->stream_info.next_free_stream_id ==
 	    kshark_ctx->stream_info.array_size) {
-		size_t new_size = 2 * kshark_ctx->stream_info.array_size;
-		struct kshark_data_stream **streams_tmp;
-
-		streams_tmp = realloc(kshark_ctx->stream,
-				      new_size * sizeof(*kshark_ctx->stream));
-		if (!streams_tmp)
+		if (!KS_DOUBLE_SIZE(kshark_ctx->stream,
+				    kshark_ctx->stream_info.array_size))
 			return -ENOMEM;
-
-		kshark_ctx->stream = streams_tmp;
-		kshark_ctx->stream_info.array_size = new_size;
 	}
 
 	stream = kshark_stream_alloc();
tests/libkshark-tests.cpp +20 -0
index 27c1171..d0a3594 100644
@@ -46,3 +46,23 @@ BOOST_AUTO_TEST_CASE(add_remove_streams)
 
 	kshark_close_all(kshark_ctx);
 }
+
+#define ARRAY_DEFAULT_SIZE	1000
+BOOST_AUTO_TEST_CASE(doule_size_macro)
+{
+	int i, n = ARRAY_DEFAULT_SIZE;
+	int *arr = (int *) malloc(n * sizeof(*arr));
+	bool ok;
+
+	for (i = 0; i < n; ++i)
+		arr[i] = i;
+
+	ok = KS_DOUBLE_SIZE(arr, n);
+	BOOST_CHECK_EQUAL(ok, true);
+	BOOST_CHECK_EQUAL(n, ARRAY_DEFAULT_SIZE * 2);
+
+	for (i = 0; i < ARRAY_DEFAULT_SIZE; ++i)
+		BOOST_CHECK_EQUAL(arr[i], i);
+	for (; i < n; ++i)
+		BOOST_CHECK_EQUAL(arr[i], 0);
+}
Signature · 1 line
2.25.1
#2 lore raw

PATCH v3 2/6 kernel-shark: Add kshark_data_container to libkshark

In reply to Yordan Karadzhov (VMware)

We add an infrastructure for recording the data from a particular trace event field during data loading. The goal is to avoid the use of the expensive "read_event_field" operation in the case when the value of the field is needed during the visualization processing (in plugins for example).

---
 src/libkshark.c           | 147 ++++++++++++++++++++++++++++++++++++++
 src/libkshark.h           |  43 +++++++++++
 tests/libkshark-tests.cpp |  37 ++++++++++
 3 files changed, 227 insertions(+)
src/libkshark.c +147 -0
index 0e2ce7a..a54fdd5 100644
@@ -2110,3 +2110,150 @@ kshark_merge_data_matrices(struct kshark_matrix_data_set *buffers, int n_buffers
  end:
 	return merged_data;
 }
+
+/** @brief Allocate memory for kshark_data_container. */
+struct kshark_data_container *kshark_init_data_container()
+{
+	struct kshark_data_container *container;
+
+	container = calloc(1, sizeof(*container));
+	if (!container)
+		goto fail;
+
+	container->data = calloc(KS_CONTAINER_DEFAULT_SIZE,
+				  sizeof(*container->data));
+
+	if (!container->data)
+		goto fail;
+
+	container->capacity = KS_CONTAINER_DEFAULT_SIZE;
+	container->sorted = false;
+
+	return container;
+
+ fail:
+	fprintf(stderr, "Failed to allocate memory for data container.\n");
+	kshark_free_data_container(container);
+	return NULL;
+}
+
+/**
+ * @brief Free the memory allocated for a kshark_data_container
+ * @param container: Input location for the kshark_data_container object.
+ */
+void kshark_free_data_container(struct kshark_data_container *container)
+{
+	if (!container)
+		return;
+
+	for (ssize_t i = 0; i < container->size; ++i)
+		free(container->data[i]);
+
+	free(container->data);
+	free(container);
+}
+
+/**
+ * @brief Append data field value to a kshark_data_container
+ * @param container: Input location for the kshark_data_container object.
+ * @param entry: The entry that needs addition data field value.
+ * @param field: The value of data field to be added.
+ *
+ * @returns The size of the container after the addition.
+ */
+ssize_t kshark_data_container_append(struct kshark_data_container *container,
+				     struct kshark_entry *entry, int64_t field)
+{
+	struct kshark_data_field_int64 *data_field;
+
+	if (container->capacity == container->size) {
+		if (!KS_DOUBLE_SIZE(container->data,
+				    container->capacity))
+			return -ENOMEM;
+	}
+
+	data_field = malloc(sizeof(*data_field));
+	data_field->entry = entry;
+	data_field->field = field;
+	container->data[container->size++] = data_field;
+
+	return container->size;
+}
+
+static int compare_time_dc(const void* a, const void* b)
+{
+	const struct kshark_data_field_int64 *field_a, *field_b;
+
+	field_a = *(const struct kshark_data_field_int64 **) a;
+	field_b = *(const struct kshark_data_field_int64 **) b;
+
+	if (field_a->entry->ts > field_b->entry->ts)
+		return 1;
+
+	if (field_a->entry->ts < field_b->entry->ts)
+		return -1;
+
+	return 0;
+}
+
+/**
+ * @brief Sort in time the records in kshark_data_container. The container is
+ *	  resized in order to free the unused memory capacity.
+ *
+ * @param container: Input location for the kshark_data_container object.
+ */
+void kshark_data_container_sort(struct kshark_data_container *container)
+{
+	struct kshark_data_field_int64	**data_tmp;
+
+	qsort(container->data, container->size,
+	      sizeof(struct kshark_data_field_int64 *),
+	      compare_time_dc);
+
+	container->sorted = true;
+
+	data_tmp = realloc(container->data,
+			   container->size * sizeof(*container->data));
+
+	if (!data_tmp)
+		return;
+
+	container->data = data_tmp;
+	container->capacity = container->size;
+}
+
+/**
+ * @brief Binary search inside a time-sorted array of kshark_data_field_int64.
+ *
+ * @param time: The value of time to search for.
+ * @param data: Input location for the data.
+ * @param l: Array index specifying the lower edge of the range to search in.
+ * @param h: Array index specifying the upper edge of the range to search in.
+ *
+ * @returns On success, the index of the first kshark_data_field_int64 inside
+ *	    the range, having a timestamp equal or bigger than "time".
+ *	    If all fields inside the range have timestamps greater than "time"
+ *	    the function returns BSEARCH_ALL_GREATER (negative value).
+ *	    If all fields inside the range have timestamps smaller than "time"
+ *	    the function returns BSEARCH_ALL_SMALLER (negative value).
+ */
+ssize_t kshark_find_entry_field_by_time(int64_t time,
+					struct kshark_data_field_int64 **data,
+					size_t l, size_t h)
+{
+	size_t mid;
+
+	if (data[l]->entry->ts > time)
+		return BSEARCH_ALL_GREATER;
+
+	if (data[h]->entry->ts < time)
+		return BSEARCH_ALL_SMALLER;
+
+	/*
+	 * After executing the BSEARCH macro, "l" will be the index of the last
+	 * entry having timestamp < time and "h" will be the index of the first
+	 * entry having timestamp >= time.
+	 */
+	BSEARCH(h, l, data[mid]->entry->ts < time);
+	return h;
+}
src/libkshark.h +43 -0

49 lines not on this page · raw message

tests/libkshark-tests.cpp +37 -0
index d0a3594..8a5dcf1 100644
@@ -66,3 +66,40 @@ BOOST_AUTO_TEST_CASE(doule_size_macro)
 	for (; i < n; ++i)
 		BOOST_CHECK_EQUAL(arr[i], 0);
 }
+
+#define N_VALUES	2 * KS_CONTAINER_DEFAULT_SIZE + 1
+#define MAX_TS		100000
+BOOST_AUTO_TEST_CASE(fill_data_container)
+{
+	struct kshark_data_container *data = kshark_init_data_container();
+	struct kshark_entry entries[N_VALUES];
+	int64_t i, ts_last(0);
+
+	BOOST_CHECK_EQUAL(data->capacity, KS_CONTAINER_DEFAULT_SIZE);
+
+	for (i = 0; i < N_VALUES; ++i) {
+		entries[i].ts = rand() % MAX_TS;
+		kshark_data_container_append(data, &entries[i], 10 - entries[i].ts);
+	}
+
+	BOOST_CHECK_EQUAL(data->size, N_VALUES);
+	BOOST_CHECK_EQUAL(data->capacity, 4 * KS_CONTAINER_DEFAULT_SIZE);
+
+	kshark_data_container_sort(data);
+	BOOST_CHECK_EQUAL(data->capacity, N_VALUES);
+	for (i = 0; i < N_VALUES; ++i) {
+		BOOST_REQUIRE(data->data[i]->entry->ts >= ts_last);
+		BOOST_CHECK_EQUAL(data->data[i]->entry->ts,
+				  10 - data->data[i]->field);
+
+		ts_last = data->data[i]->entry->ts;
+	}
+
+	i = kshark_find_entry_field_by_time(MAX_TS / 2, data->data,
+					    0, N_VALUES - 1);
+
+	BOOST_REQUIRE(data->data[i - 1]->entry->ts < MAX_TS / 2);
+	BOOST_REQUIRE(data->data[i]->entry->ts >= MAX_TS / 2);
+
+	kshark_free_data_container(data);
+}
Signature · 1 line
2.25.1
#3 lore raw

PATCH v3 3/6 kernel-shark: Add KS_DEFINE_PLUGIN_CONTEXT macro

In reply to Yordan Karadzhov (VMware)

When we implement a KernelShark plugins, we often need a way to define a structure that can hold context data that is visible only inside the plugin and that has specific values for each data stream. The tricky part here is that the total number of data streams and the IDs of the active streams are dynamic quantities that can change as the user adds or removes data streams. The macro defines an interface of functions that will be useful for the plugin developer, helping to directly use context objects, without caring for the complications due to the dynamic configuration of active data streams.

---
 src/libkshark-plugin.h    | 41 +++++++++++++++++++++++++++++++++++++++
 tests/libkshark-tests.cpp | 32 ++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
src/libkshark-plugin.h +41 -0
index 4104357..c62086f 100644
@@ -363,6 +363,47 @@ int kshark_handle_all_dpis(struct kshark_data_stream *stream,
 	__ok;								\
 })									\
 
+/** General purpose macro defining methods for adding plugin context. */
+#define KS_DEFINE_PLUGIN_CONTEXT(type)					\
+static type **__context_handler;					\
+static ssize_t __n_streams = -1;					\
+static inline type *__init(int sd)					\
+{									\
+	type *obj;							\
+	if (__n_streams < 0 && sd < KS_DEFAULT_NUM_STREAMS) {		\
+		__context_handler =					\
+			(type **) calloc(KS_DEFAULT_NUM_STREAMS,	\
+					 sizeof(*__context_handler));	\
+		if (!__context_handler)					\
+			return NULL;					\
+		__n_streams = KS_DEFAULT_NUM_STREAMS;			\
+	} else if (sd >= __n_streams) {					\
+		if (!KS_DOUBLE_SIZE(__context_handler,			\
+				    __n_streams))			\
+			return NULL;					\
+	}								\
+	assert(__context_handler[sd] == NULL);				\
+	obj = (type *) calloc(1, sizeof(*obj));				\
+	__context_handler[sd] = obj;					\
+	return obj;							\
+}									\
+static inline void __close(int sd)					\
+{									\
+	if (sd < 0) {							\
+		free(__context_handler);				\
+		__n_streams = -1;					\
+		return;							\
+	}								\
+	free(__context_handler[sd]);					\
+	__context_handler[sd] = NULL;					\
+}									\
+static inline type *__get_context(int sd)				\
+{									\
+	if (sd < 0 || sd >= __n_streams)				\
+		return NULL;						\
+	return __context_handler[sd];					\
+}									\
+
 #ifdef __cplusplus
 }
 #endif // __cplusplus
tests/libkshark-tests.cpp +32 -0
index 8a5dcf1..780a3fa 100644
@@ -10,6 +10,7 @@
 
 // KernelShark
 #include "libkshark.h"
+#include "libkshark-plugin.h"
 
 #define N_TEST_STREAMS	1000
 
@@ -103,3 +104,34 @@ BOOST_AUTO_TEST_CASE(fill_data_container)
 
 	kshark_free_data_container(data);
 }
+
+struct test_context {
+	int a;
+	char b;
+};
+
+KS_DEFINE_PLUGIN_CONTEXT(struct test_context);
+
+BOOST_AUTO_TEST_CASE(init_close_plugin)
+{
+	struct test_context *ctx;
+	int i;
+
+	for (i = 0; i < N_TEST_STREAMS; ++i) {
+		ctx = __init(i);
+		ctx->a = i * 10;
+		ctx->b = 'z';
+	}
+
+	for (i = 0; i < N_TEST_STREAMS; ++i) {
+		ctx = __get_context(i);
+		BOOST_REQUIRE(ctx != NULL);
+		BOOST_CHECK_EQUAL(ctx->a, i * 10);
+		BOOST_CHECK_EQUAL(ctx->b, 'z');
+
+		__close(i);
+		BOOST_REQUIRE(__get_context(i) == NULL);
+	}
+
+	__close(-1);
+}
Signature · 1 line
2.25.1
#4 lore raw

PATCH v3 4/6 kernel-shark: Start using C++17

In reply to Yordan Karadzhov (VMware)

Update to a more recent version of the C++ standard. The particular motivation to do this change now is because, we would like to use "auto" as a function parameter type for lambda functions.

---
 CMakeLists.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
CMakeLists.txt +6 -2
index 9abacd0..dd62091 100644
@@ -62,8 +62,12 @@ find_package (Boost COMPONENTS unit_test_framework)
 set(LIBRARY_OUTPUT_PATH    "${KS_DIR}/lib")
 set(EXECUTABLE_OUTPUT_PATH "${KS_DIR}/bin")
 
-set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -Wall -pthread -fPIC")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11 -pthread -fPIC")
+set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -Wall -pthread -fPIC -fno-common")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pthread -fPIC -fno-common")
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
 
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE RelWithDebInfo)
Signature · 1 line
2.25.1
#5 lore raw

PATCH v3 5/6 kernel-shark: Add plotting methods to KsPlugins

In reply to Yordan Karadzhov (VMware)

We add generic methods for visualizing the value of a given trace event field or the relation between two given trace event field. Those methods are taking advantage of the stored values of the event fields in kshark_data_container objects and allow the visualization plugin to process the data orders of magnitude faster.

---
 src/CMakeLists.txt |   3 +-
 src/KsPlugins.cpp  | 416 +++++++++++++++++++++++++++++++++++++++++++++
 src/KsPlugins.hpp  |  48 ++++++
 3 files changed, 466 insertions(+), 1 deletion(-)
 create mode 100644 src/KsPlugins.cpp
src/CMakeLists.txt +2 -1
index e35b436..588cccd 100644
@@ -41,7 +41,8 @@ if (OPENGL_FOUND)
 
     message(STATUS "libkshark-plot")
     add_library(kshark-plot  SHARED  libkshark-plot.c
-                                     KsPlotTools.cpp)
+                                     KsPlotTools.cpp
+                                     KsPlugins.cpp)
 
     target_link_libraries(kshark-plot  kshark
                                        ${GLUT_LIBRARY}
src/KsPlugins.cpp new file +416 -0

416 lines not on this page · raw message

src/KsPlugins.hpp +48 -0

58 lines not on this page · raw message

Signature · 1 line
2.25.1
#6 lore raw

PATCH v3 6/6 kernel-shark: Speed-up the sched_events plugin

In reply to Yordan Karadzhov (VMware)

General revision of the sched_events plugin that achieves much faster processing of the wake-up latency, by using the new generic methods for visualization of event's data field.

---
 src/CMakeLists.txt          |   2 +-
 src/libkshark-plugin.h      |   1 +
 src/libkshark-tepdata.c     |  34 ++++
 src/libkshark-tepdata.h     |   7 +
 src/plugins/CMakeLists.txt  |  11 +-
 src/plugins/SchedEvents.cpp | 310 +++++++---------------------
 src/plugins/sched_events.c  | 393 +++++++++++-------------------------
 src/plugins/sched_events.h  |  50 ++---
 8 files changed, 260 insertions(+), 548 deletions(-)
src/CMakeLists.txt +1 -1
index 588cccd..980e802 100644
@@ -135,7 +135,7 @@ if (Qt5Widgets_FOUND AND Qt5Network_FOUND)
 
 endif (Qt5Widgets_FOUND AND Qt5Network_FOUND)
 
-# add_subdirectory(plugins)
+add_subdirectory(plugins)
 
 find_program(DO_AS_ROOT pkexec)
 
src/libkshark-plugin.h +1 -0
index c62086f..c110616 100644
@@ -19,6 +19,7 @@ extern "C" {
 // C
 #include <stdint.h>
 #include <stdbool.h>
+#include <assert.h>
 
 /* Quiet warnings over documenting simple structures */
 //! @cond Doxygen_Suppress
src/libkshark-tepdata.c +34 -0
index 724cff2..d4be052 100644
@@ -1912,3 +1912,37 @@ int kshark_tep_find_top_stream(struct kshark_context *kshark_ctx,
 
 	return top_stream->stream_id;
 }
+
+static bool find_wakeup_event(struct tep_handle *tep, const char *wakeup_name,
+			      struct tep_event **waking_event_ptr)
+{
+	*waking_event_ptr = tep_find_event_by_name(tep, "sched", wakeup_name);
+
+	return (*waking_event_ptr)? true : false;
+}
+
+/**
+ * @brief Search the available trace events and retrieve a definition of
+ *	  a waking_event.
+ *
+ * @param tep: Input location for the the Page event object.
+ * @param waking_event_ptr: Output location for the the waking_event object.
+ *
+ * @returns True on success, otherwise False.
+ */
+bool define_wakeup_event(struct tep_handle *tep,
+			 struct tep_event **waking_event_ptr)
+{
+	bool wakeup_found;
+
+	wakeup_found = find_wakeup_event(tep, "sched_wakeup",
+					 waking_event_ptr);
+
+	wakeup_found |= find_wakeup_event(tep, "sched_wakeup_new",
+					  waking_event_ptr);
+
+	wakeup_found |= find_wakeup_event(tep, "sched_waking",
+					  waking_event_ptr);
+
+	return wakeup_found;
+}
src/libkshark-tepdata.h +7 -0
index 46b18c8..1b955be 100644
@@ -105,6 +105,13 @@ int kshark_tep_find_top_stream(struct kshark_context *kshark_ctx,
 
 bool kshark_tep_is_top_stream(struct kshark_data_stream *stream);
 
+struct tep_event;
+
+struct tep_format_field;
+
+bool define_wakeup_event(struct tep_handle *tep,
+			 struct tep_event **wakeup_event);
+
 #ifdef __cplusplus
 }
 #endif
src/plugins/CMakeLists.txt +5 -6
index 2da73f8..108bc5f 100644
@@ -20,14 +20,13 @@ endfunction()
 set(PLUGIN_LIST "")
 BUILD_PLUGIN(NAME sched_events
              SOURCE sched_events.c SchedEvents.cpp)
-list(APPEND PLUGIN_LIST "sched_events default") # This plugin will be loaded by default
-# list(APPEND PLUGIN_LIST "sched_events") # This plugin isn't loaded by default
+list(APPEND PLUGIN_LIST "sched_events")
 
-BUILD_PLUGIN(NAME missed_events
-             SOURCE missed_events.c MissedEvents.cpp)
-list(APPEND PLUGIN_LIST "missed_events default") # This plugin will be loaded by default
+# BUILD_PLUGIN(NAME missed_events
+#              SOURCE missed_events.c MissedEvents.cpp)
+# list(APPEND PLUGIN_LIST "missed_events")
 
-install(TARGETS sched_events missed_events
+install(TARGETS ${PLUGIN_LIST}
         LIBRARY DESTINATION ${KS_PLUGIN_INSTALL_PREFIX}
         COMPONENT kernelshark)
 
src/plugins/SchedEvents.cpp +73 -237

371 lines not on this page · raw message

src/plugins/sched_events.c +122 -271

484 lines not on this page · raw message

src/plugins/sched_events.h +17 -33

88 lines not on this page · raw message

Signature · 1 line
2.25.1