Skip to contentSkip to conversation

lib: add extra calibrated runtime to slow tests

PATCH In review Li Wang 2 participants 4 messages Nov 27, 2024 – Dec 6, 2024 Library code

lore
Share & tools
lore

Li Wang

author
#0 lore raw

PATCH 1/2 lib: add extra calibrated runtime to slow tests

This patch inspired by Cyril's work ec14f4572d ("sched:
starvation: Autocallibrate the timeout"), introduces a CPU calibration mechanism to enhance test runtime calculations.

The new tst_callibrate function measures CPU performance, and in combination with tst_kconfig_debug_matches which detects enabled debug-kernel configurations, computes an extra_runtime adjustment.

The multiply_runtime function now incorporates this adjustment to account for debug overhead, reducing the likelihood of timeouts while maintaining adaptability for slower systems.

---
 include/tst_kconfig.h | 32 ++++++++++++++++++++++++++++++++
 include/tst_timer.h   | 30 ++++++++++++++++++++++++++++++
 lib/tst_test.c        |  5 ++++-
 3 files changed, 66 insertions(+), 1 deletion(-)
include/tst_kconfig.h +32 -0
index 23f807409..5746925df 100644
@@ -98,4 +98,36 @@ struct tst_kcmdline_var {
  */
 void tst_kcmdline_parse(struct tst_kcmdline_var params[], size_t params_len);
 
+/*
+ * List of debug-kernel config options that may degrade performance when enabled.
+ */
+static const char * const tst_kconf_debug_options[][2] = {
+	{"CONFIG_PROVE_LOCKING=y", NULL},
+	{"CONFIG_LOCKDEP=y", NULL},
+	{"CONFIG_DEBUG_SPINLOCK=y", NULL},
+	{"CONFIG_DEBUG_RT_MUTEXES=y", NULL},
+	{"CONFIG_DEBUG_MUTEXES=y", NULL},
+	{"CONFIG_DEBUG_PAGEALLOC=y", NULL},
+	{"CONFIG_KASAN=y", NULL},
+	{"CONFIG_SLUB_RCU_DEBUG=y", NULL},
+	{"CONFIG_TRACE_IRQFLAGS=y", NULL},
+	{"CONFIG_LATENCYTOP=y", NULL},
+	{"CONFIG_DEBUG_NET=y", NULL},
+	{"CONFIG_EXT4_DEBUG=y", NULL},
+	{"CONFIG_QUOTA_DEBUG=y", NULL},
+	{"CONFIG_FAULT_INJECTION=y", NULL},
+	{"CONFIG_DEBUG_OBJECTS=y", NULL},
+	{NULL, NULL}
+};
+
+static inline int tst_kconfig_debug_matches(void)
+{
+	int i, num = 1;
+
+	for (i = 0; tst_kconf_debug_options[i][0] != NULL; i++)
+		num += !tst_kconfig_check(tst_kconf_debug_options[i]);
+
+	return num;
+}
+
 #endif	/* TST_KCONFIG_H__ */
include/tst_timer.h +30 -0
index 6fb940020..268fc8389 100644
@@ -17,6 +17,7 @@
 #include <mqueue.h>
 #include <time.h>
 #include "tst_test.h"
+#include "tst_clocks.h"
 #include "lapi/common_timers.h"
 #include "lapi/posix_types.h"
 #include "lapi/syscalls.h"
@@ -1074,4 +1075,33 @@ static inline long long tst_timer_elapsed_us(void)
 	return tst_timespec_to_us(tst_timer_elapsed());
 }
 
+#define CALLIBRATE_LOOPS 120000000
+
+/*
+ * Measures the time taken by the CPU to perform a specified
+ * number of empty loops for calibration.
+ */
+static inline int tst_callibrate(void)
+{
+	int i;
+	struct timespec start, stop;
+	long long diff;
+
+	for (i = 0; i < CALLIBRATE_LOOPS; i++)
+		__asm__ __volatile__ ("" : "+g" (i) : :);
+
+	tst_clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+
+	for (i = 0; i < CALLIBRATE_LOOPS; i++)
+		__asm__ __volatile__ ("" : "+g" (i) : :);
+
+	tst_clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
+
+	diff = tst_timespec_diff_us(stop, start);
+
+	tst_res(TINFO, "CPU did %i loops in %llius", CALLIBRATE_LOOPS, diff);
+
+	return diff;
+}
+
 #endif /* TST_TIMER */
lib/tst_test.c +4 -1
index 8db554dea..296683ffb 100644
@@ -549,13 +549,16 @@ static void parse_mul(float *mul, const char *env_name, float min, float max)
 static int multiply_runtime(int max_runtime)
 {
 	static float runtime_mul = -1;
+	static int extra_runtime = 0;
 
 	if (max_runtime <= 0)
 		return max_runtime;
 
 	parse_mul(&runtime_mul, "LTP_RUNTIME_MUL", 0.0099, 100);
 
-	return max_runtime * runtime_mul;
+	extra_runtime = (tst_callibrate() / 1000) * tst_kconfig_debug_matches();
+
+	return (max_runtime + extra_runtime) * runtime_mul;
 }
 
 static struct option {
Signature · 5 lines
2.47.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
author
#1 lore raw

PATCH 2/2 starvation: define the max_runtime

---
 .../kernel/sched/cfs-scheduler/starvation.c   | 38 +------------------
 1 file changed, 1 insertion(+), 37 deletions(-)
testcases/kernel/sched/cfs-scheduler/starvation.c +1 -37
index e707e0865..cd6739a24 100644
@@ -21,38 +21,9 @@
 #include <sched.h>
 
 #include "tst_test.h"
-#include "tst_safe_clocks.h"
-#include "tst_timer.h"
 
 static char *str_loop;
 static long loop = 1000000;
-static char *str_timeout;
-static int timeout;
-
-#define CALLIBRATE_LOOPS 120000000
-
-static int callibrate(void)
-{
-	int i;
-	struct timespec start, stop;
-	long long diff;
-
-	for (i = 0; i < CALLIBRATE_LOOPS; i++)
-		__asm__ __volatile__ ("" : "+g" (i) : :);
-
-	SAFE_CLOCK_GETTIME(CLOCK_MONOTONIC_RAW, &start);
-
-	for (i = 0; i < CALLIBRATE_LOOPS; i++)
-		__asm__ __volatile__ ("" : "+g" (i) : :);
-
-	SAFE_CLOCK_GETTIME(CLOCK_MONOTONIC_RAW, &stop);
-
-	diff = tst_timespec_diff_us(stop, start);
-
-	tst_res(TINFO, "CPU did %i loops in %llius", CALLIBRATE_LOOPS, diff);
-
-	return diff;
-}
 
 static int wait_for_pid(pid_t pid)
 {
@@ -102,13 +73,6 @@ static void setup(void)
 
 	if (tst_parse_long(str_loop, &loop, 1, LONG_MAX))
 		tst_brk(TBROK, "Invalid number of loop number '%s'", str_loop);
-
-	if (tst_parse_int(str_timeout, &timeout, 1, INT_MAX))
-		tst_brk(TBROK, "Invalid number of timeout '%s'", str_timeout);
-	else
-		timeout = callibrate() / 1000;
-
-	tst_set_max_runtime(timeout);
 }
 
 static void handler(int sig LTP_ATTRIBUTE_UNUSED)
@@ -158,8 +122,8 @@ static struct tst_test test = {
 	.forks_child = 1,
 	.options = (struct tst_option[]) {
 		{"l:", &str_loop, "Number of loops (default 2000000)"},
-		{"t:", &str_timeout, "Max timeout (default 240s)"},
 		{}
 	},
+	.max_runtime = 120,
 	.needs_checkpoints = 1,
 };
Signature · 5 lines
2.47.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
#2 lore raw
In reply to Li Wang

Hi!

Quoted 88 lines
include/tst_kconfig.h +32 -0
+/*
+ * List of debug-kernel config options that may degrade performance when enabled.
+ */
+static const char * const tst_kconf_debug_options[][2] = {
+	{"CONFIG_PROVE_LOCKING=y", NULL},
+	{"CONFIG_LOCKDEP=y", NULL},
+	{"CONFIG_DEBUG_SPINLOCK=y", NULL},
+	{"CONFIG_DEBUG_RT_MUTEXES=y", NULL},
+	{"CONFIG_DEBUG_MUTEXES=y", NULL},
+	{"CONFIG_DEBUG_PAGEALLOC=y", NULL},
+	{"CONFIG_KASAN=y", NULL},
+	{"CONFIG_SLUB_RCU_DEBUG=y", NULL},
+	{"CONFIG_TRACE_IRQFLAGS=y", NULL},
+	{"CONFIG_LATENCYTOP=y", NULL},
+	{"CONFIG_DEBUG_NET=y", NULL},
+	{"CONFIG_EXT4_DEBUG=y", NULL},
+	{"CONFIG_QUOTA_DEBUG=y", NULL},
+	{"CONFIG_FAULT_INJECTION=y", NULL},
+	{"CONFIG_DEBUG_OBJECTS=y", NULL},
+	{NULL, NULL}
+};
+
+static inline int tst_kconfig_debug_matches(void)
+{
+	int i, num = 1;
+
+	for (i = 0; tst_kconf_debug_options[i][0] != NULL; i++)
+		num += !tst_kconfig_check(tst_kconf_debug_options[i]);
+
+	return num;
+}
+
 #endif	/* TST_KCONFIG_H__ */
include/tst_timer.h +30 -0
index 6fb940020..268fc8389 100644
@@ -17,6 +17,7 @@
 #include <mqueue.h>
 #include <time.h>
 #include "tst_test.h"
+#include "tst_clocks.h"
 #include "lapi/common_timers.h"
 #include "lapi/posix_types.h"
 #include "lapi/syscalls.h"
@@ -1074,4 +1075,33 @@ static inline long long tst_timer_elapsed_us(void)
 	return tst_timespec_to_us(tst_timer_elapsed());
 }
 
+#define CALLIBRATE_LOOPS 120000000
+
+/*
+ * Measures the time taken by the CPU to perform a specified
+ * number of empty loops for calibration.
+ */
+static inline int tst_callibrate(void)
+{
+	int i;
+	struct timespec start, stop;
+	long long diff;
+
+	for (i = 0; i < CALLIBRATE_LOOPS; i++)
+		__asm__ __volatile__ ("" : "+g" (i) : :);
+
+	tst_clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+
+	for (i = 0; i < CALLIBRATE_LOOPS; i++)
+		__asm__ __volatile__ ("" : "+g" (i) : :);
+
+	tst_clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
+
+	diff = tst_timespec_diff_us(stop, start);
+
+	tst_res(TINFO, "CPU did %i loops in %llius", CALLIBRATE_LOOPS, diff);
+
+	return diff;
+}
+
 #endif /* TST_TIMER */
lib/tst_test.c +1 -0
index 8db554dea..296683ffb 100644
@@ -549,13 +549,16 @@ static void parse_mul(float *mul, const char *env_name, float min, float max)
 static int multiply_runtime(int max_runtime)
 {
 	static float runtime_mul = -1;
+	static int extra_runtime = 0;

lib/tst_test.c static void parse_mul(float *mul, const char *env_name, float min, float max) · line 561

Quoted 10 lines
lib/tst_test.c +3 -1
 
 	if (max_runtime <= 0)
 		return max_runtime;
 
 	parse_mul(&runtime_mul, "LTP_RUNTIME_MUL", 0.0099, 100);
 
-	return max_runtime * runtime_mul;
+	extra_runtime = (tst_callibrate() / 1000) * tst_kconfig_debug_matches();
+
+	return (max_runtime + extra_runtime) * runtime_mul;

I was hoping for a simpler calculation something as:

	if (any_kconfig_debug_enabled())
		max_runtime *= 4;

Or something along these lines. I do not think that the kernel will get slower with each debug option enabled. And we are not looking for a tight fit either, so the callbration loop does not make much sense here.

It also makes sense that a debug kernel will be N times slower, where the exact N would be hard to find out, but in this case we just need a reasonable upper bound.

And this would work nicely for the starvation test too, since we calculate a tight fit there first, then we would multiply it by the "debug kernel slowness" multiplier. There would be no need to modify the starvation test in that case either (it may make sense to set the runtime of the test to -1 so that the metadata will have an information that the test runtime is unbound until we actually execute it though).

Signature · 5 lines
Cyril Hrubis
[email protected]

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
author
#3 lore raw
Quoted 100 lines · Cyril Hrubis

Hi!

Quoted 2 lines

+/*
+ * List of debug-kernel config options that may degrade performance

when enabled.

Quoted 62 lines

+ */

include/tst_kconfig.h +29 -0
+static const char * const tst_kconf_debug_options[][2] = {
+     {"CONFIG_PROVE_LOCKING=y", NULL},
+     {"CONFIG_LOCKDEP=y", NULL},
+     {"CONFIG_DEBUG_SPINLOCK=y", NULL},
+     {"CONFIG_DEBUG_RT_MUTEXES=y", NULL},
+     {"CONFIG_DEBUG_MUTEXES=y", NULL},
+     {"CONFIG_DEBUG_PAGEALLOC=y", NULL},
+     {"CONFIG_KASAN=y", NULL},
+     {"CONFIG_SLUB_RCU_DEBUG=y", NULL},
+     {"CONFIG_TRACE_IRQFLAGS=y", NULL},
+     {"CONFIG_LATENCYTOP=y", NULL},
+     {"CONFIG_DEBUG_NET=y", NULL},
+     {"CONFIG_EXT4_DEBUG=y", NULL},
+     {"CONFIG_QUOTA_DEBUG=y", NULL},
+     {"CONFIG_FAULT_INJECTION=y", NULL},
+     {"CONFIG_DEBUG_OBJECTS=y", NULL},
+     {NULL, NULL}
+};
+
+static inline int tst_kconfig_debug_matches(void)
+{
+     int i, num = 1;
+
+     for (i = 0; tst_kconf_debug_options[i][0] != NULL; i++)
+             num += !tst_kconfig_check(tst_kconf_debug_options[i]);
+
+     return num;
+}
+
 #endif       /* TST_KCONFIG_H__ */
include/tst_timer.h +16 -0
index 6fb940020..268fc8389 100644
@@ -17,6 +17,7 @@
 #include <mqueue.h>
 #include <time.h>
 #include "tst_test.h"
+#include "tst_clocks.h"
 #include "lapi/common_timers.h"
 #include "lapi/posix_types.h"
 #include "lapi/syscalls.h"
@@ -1074,4 +1075,33 @@ static inline long long tst_timer_elapsed_us(void)
      return tst_timespec_to_us(tst_timer_elapsed());
 }
 
+#define CALLIBRATE_LOOPS 120000000
+
+/*
+ * Measures the time taken by the CPU to perform a specified
+ * number of empty loops for calibration.
+ */
+static inline int tst_callibrate(void)
+{
+     int i;
+     struct timespec start, stop;
+     long long diff;
+
+     for (i = 0; i < CALLIBRATE_LOOPS; i++)
+             __asm__ __volatile__ ("" : "+g" (i) : :);
+

include/tst_timer.h static inline long long tst_timer_elapsed_us(void) · line 1102

Quoted 10 lines
include/tst_timer.h +10 -0
+     tst_clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+
+     for (i = 0; i < CALLIBRATE_LOOPS; i++)
+             __asm__ __volatile__ ("" : "+g" (i) : :);
+
+     tst_clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
+
+     diff = tst_timespec_diff_us(stop, start);
+
+     tst_res(TINFO, "CPU did %i loops in %llius", CALLIBRATE_LOOPS,

diff);

lib/tst_test.c static void parse_mul(float *mul, const char

Quoted 10 lines
include/tst_timer.h +4 -0
+
+     return diff;
+}
+
 #endif /* TST_TIMER */
lib/tst_test.c +0 -0
index 8db554dea..296683ffb 100644
@@ -549,13 +549,16 @@ static void parse_mul(float *mul, const char

*env_name, float min, float max)

lib/tst_test.c static void parse_mul(float *mul, const char *env_name, float min, float max)

Quoted 12 lines
lib/tst_test.c +2 -1
 static int multiply_runtime(int max_runtime)
 {
      static float runtime_mul = -1;
+     static int extra_runtime = 0;
 
      if (max_runtime <= 0)
              return max_runtime;
 
      parse_mul(&runtime_mul, "LTP_RUNTIME_MUL", 0.0099, 100);
 
-     return max_runtime * runtime_mul;
+     extra_runtime = (tst_callibrate() / 1000) *
Quoted 9 lines

tst_kconfig_debug_matches();

lib/tst_test.c static void parse_mul(float *mul, const char *env_name, float min, float max) · line 561

Quoted 2 lines
lib/tst_test.c +2 -0
+
+     return (max_runtime + extra_runtime) * runtime_mul;

I was hoping for a simpler calculation something as:

        if (any_kconfig_debug_enabled())
                max_runtime *= 4;

Ok, if we do this, the debug configuration set tst_kconf_debug_options should be confirmed to work with debug kernels on the most popular distributions (e.g. OpenSuse, Ubuntu), and I can help check if this works on Fedora family systems.

Otherwise any_kconfig_debug_enabled() will be limited by the Linux distribution.

[Cc Shizhao who is also reporting the starvation failures to me]

Quoted 20 lines

Or something along these lines. I do not think that the kernel will get slower with each debug option enabled. And we are not looking for a tight fit either, so the callbration loop does not make much sense here.

It also makes sense that a debug kernel will be N times slower, where the exact N would be hard to find out, but in this case we just need a reasonable upper bound.

And this would work nicely for the starvation test too, since we calculate a tight fit there first, then we would multiply it by the "debug kernel slowness" multiplier. There would be no need to modify the starvation test in that case either (it may make sense to set the runtime of the test to -1 so that the metadata will have an information that the test runtime is unbound until we actually execute it though).

--
Cyril Hrubis
[email protected]

Signature · 5 lines
Regards,
Li Wang

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp