lib: tst_test: Add reproducible output.
PATCH v3 1/1 lib: tst_test: Add reproducible output.
This commit adds an environment variable LTP_REPRODUCIBLE_OUTPUT that when set to 1 or y skips printing parts of the test messages that may contain data that differ on subsequent runs (e.g. pids).
With this you can run a test twice under a different conditions and check if the test codeflow was identical by simply doing diff of the outputs from the two runs.
--- doc/users/setup_tests.rst | 4 ++++ lib/tst_test.c | 33 ++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-)
doc/users/setup_tests.rst +4 -0
@@ -42,6 +42,10 @@ users. - Path to the block device to be used. C Language: ``.needs_device = 1``. Shell language: ``TST_NEEDS_DEVICE=1``. + * - LTP_REPRODUCIBLE_OUTPUT + - When set to ``1`` or ``y`` discards the actual content of the messages + printed by the test (suitable for a reproducible output). + * - LTP_SINGLE_FS_TYPE - Testing only - specifies filesystem instead all supported (for tests with ``.all_filesystems``).
lib/tst_test.c +22 -11
@@ -63,6 +63,7 @@ static int mntpoint_mounted;static int ovl_mounted;static struct timespec tst_start_time;/* valid only for test pid */ static int tdebug; +static int reproducible_output;struct results {int passed;
@@ -316,6 +317,9 @@ static void print_result(const char *file, const int lineno, int ttype, str += ret; size -= ret; +if (reproducible_output) + goto print; + ssize = size -2 ; ret =vsnprintf (str, size, fmt, va); str +=MIN (ret, ssize);
@@ -333,6 +337,7 @@ static void print_result(const char *file, const int lineno, int ttype, "Next message is too long and truncated:"); } +print:snprintf (str, size,"\n" );/* we might be called from signal handler, so use write() */
@@ -606,17 +611,18 @@ static void print_help(void)/* see doc/users/setup_tests.rst, which lists also shell API variables */ fprintf (stderr,"Environment Variables\n" );fprintf (stderr,"---------------------\n" ); -fprintf (stderr,"KCONFIG_PATH Specify kernel config file\n" ); -fprintf (stderr,"KCONFIG_SKIP_CHECK Skip kernel config check if variable set (not set by default)\n" ); -fprintf (stderr,"LTPROOT Prefix for installed LTP (default: /opt/ltp)\n" ); -fprintf (stderr,"LTP_COLORIZE_OUTPUT Force colorized output behaviour (y/1 always, n/0: never)\n" ); -fprintf (stderr,"LTP_DEV Path to the block device to be used (for .needs_device)\n" ); -fprintf (stderr,"LTP_DEV_FS_TYPE Filesystem used for testing (default: %s)\n" ,DEFAULT_FS_TYPE ); -fprintf (stderr,"LTP_SINGLE_FS_TYPE Testing only - specifies filesystem instead all supported (for .all_filesystems)\n" ); -fprintf (stderr,"LTP_TIMEOUT_MUL Timeout multiplier (must be a number >=1)\n" ); -fprintf (stderr,"LTP_RUNTIME_MUL Runtime multiplier (must be a number >=1)\n" ); -fprintf (stderr,"LTP_VIRT_OVERRIDE Overrides virtual machine detection (values: \"\"|kvm|microsoft|xen|zvm)\n" ); -fprintf (stderr,"TMPDIR Base directory for template directory (for .needs_tmpdir, default: %s)\n" ,TEMPDIR ); +fprintf (stderr,"KCONFIG_PATH Specify kernel config file\n" ); +fprintf (stderr,"KCONFIG_SKIP_CHECK Skip kernel config check if variable set (not set by default)\n" ); +fprintf (stderr,"LTPROOT Prefix for installed LTP (default: /opt/ltp)\n" ); +fprintf (stderr,"LTP_COLORIZE_OUTPUT Force colorized output behaviour (y/1 always, n/0: never)\n" ); +fprintf (stderr,"LTP_DEV Path to the block device to be used (for .needs_device)\n" ); +fprintf (stderr,"LTP_DEV_FS_TYPE Filesystem used for testing (default: %s)\n" ,DEFAULT_FS_TYPE ); +fprintf (stderr,"LTP_REPRODUCIBLE_OUTPUT When set to 1 or y discards the actual content of the messages printed by the test\n" ); +fprintf (stderr,"LTP_SINGLE_FS_TYPE Testing only - specifies filesystem instead all supported (for .all_filesystems)\n" ); +fprintf (stderr,"LTP_TIMEOUT_MUL Timeout multiplier (must be a number >=1)\n" ); +fprintf (stderr,"LTP_RUNTIME_MUL Runtime multiplier (must be a number >=1)\n" ); +fprintf (stderr,"LTP_VIRT_OVERRIDE Overrides virtual machine detection (values: \"\"|kvm|microsoft|xen|zvm)\n" ); +fprintf (stderr,"TMPDIR Base directory for template directory (for .needs_tmpdir, default: %s)\n" ,TEMPDIR );fprintf (stderr,"\n" );fprintf (stderr,"Timeout and runtime\n" );
@@ -1298,6 +1304,7 @@ static const char *default_fs_type(void)static void do_setup (int argc,char *argv[]) {char *tdebug_env =getenv ("LTP_ENABLE_DEBUG" ); +char *reproducible_env =getenv ("LTP_REPRODUCIBLE_OUTPUT" );if (!tst_test)tst_brk (TBROK ,"No tests to run" );
@@ -1316,6 +1323,10 @@ static void do_setup(int argc, char *argv[])if (tst_test->supported_archs && !tst_is_on_arch (tst_test->supported_archs))tst_brk (TCONF ,"This arch '%s' is not supported for test!" , tst_arch.name); +if (reproducible_env && + (!strcmp (reproducible_env,"1" ) || !strcmp (reproducible_env,"y" ))) + reproducible_output =1 ; +assert_test_fn ();TCID = tid =get_tid (argv);
Signature · 5 lines
2.49.0 -- Mailing list info: https://lists.linux.it/listinfo/ltp
Hi Cyril,
Quoted 7 lines
This commit adds an environment variable
LTP_REPRODUCIBLE_OUTPUTthat when set to 1 or y skips printing parts of the test messages that may contain data that differ on subsequent runs (e.g. pids).With this you can run a test twice under a different conditions and check if the test codeflow was identical by simply doing diff of the outputs from the two runs.
Thanks for fixing n and 0 and updating the docs.
Kind regards,
Petr
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Hi Cyril,
+ fprintf(stderr, "LTP_REPRODUCIBLE_OUTPUT When set to 1 or y discards the actual content of the messages printed by the test\n");
nit: "When set to 1 or y" is slightly verbose to me. Maybe just "1/y discards the actual content of the messages printed by the test" ?
But anyway that's very minor, therefore I kept the same approach when adding LTP_ENABLE_DEBUG in the next patch I'm going to send now.
Kind regards,
Petr
Signature · 1 line
Mailing list info: https://lists.linux.it/listinfo/ltp
Hi!
Quoted 4 lines
lib/tst_test.c +1
-0+fprintf (stderr,"LTP_REPRODUCIBLE_OUTPUT When set to 1 or y discards the actual content of the messages printed by the test\n" );nit: "When set to 1 or y" is slightly verbose to me. Maybe just "1/y discards the actual content of the messages printed by the test" ?
I've shortened it to "Values 1 or y discard ..." and pushed the patch.
Signature · 5 lines
Cyril Hrubis [email protected] -- Mailing list info: https://lists.linux.it/listinfo/ltp