Enable Reproducible OEQA Tests
cover Enable Reproducible OEQA Tests
Enables the OEQA test for reproducible builds, and reporting of the results using resulttool
[YOCTO #13323]
Joshua Watt (5):
oeqa: reproducible: Cleanup reproducible build
oeqa: reproducible: Use subTest for packages
oeqa: Set LD_LIBRARY_PATH when executing native commands
oeqa: reproducible: Record packages in test results
oeqa: Enable reproducible build test
meta/lib/oeqa/selftest/cases/reproducible.py | 43 ++++++++++++++------ meta/lib/oeqa/utils/commands.py | 3 ++ 2 files changed, 33 insertions(+), 13 deletions(-)
Signature · 1 line
2.21.0
PATCH 1/5 oeqa: reproducible: Cleanup reproducible build
Cleans up the output from the reproducible build before building to ensure consistent results. Note that the output put is purposely left after around after the build so that non-reproducible packages can be diffed.
Signed-off-by Joshua Watt <[email protected]>
--- meta/lib/oeqa/selftest/cases/reproducible.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
meta/lib/oeqa/selftest/cases/reproducible.py +10 -3
@@ -132,16 +132,23 @@ class ReproducibleTests(OESelftestTestCase): bitbake("diffutils-native -c addto_recipe_sysroot") diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") + reproducible_tmp = os.path.join(self.topdir, 'reproducible', 'tmp') + if os.path.exists(reproducible_tmp): + bb.utils.remove(reproducible_tmp) + # Perform another build. This build should *not* share sstate or pull # from any mirrors, but sharing a DL_DIR is fine - self.write_config(textwrap.dedent('''\ - TMPDIR = "${TOPDIR}/reproducible/tmp" + self.write_config((textwrap.dedent('''\ + TMPDIR = "%s" SSTATE_DIR = "${TMPDIR}/sstate" SSTATE_MIRROR = "" - ''') + common_config) + ''') % reproducible_tmp) + common_config) vars_test = get_bb_vars(capture_vars) bitbake(' '.join(self.images)) + # NOTE: The temp directory from the reproducible build is purposely + # kept after the build so it can be diffed for debugging. + for c in self.package_classes: package_class = 'package_' + c
Signature · 1 line
2.21.0
PATCH 2/5 oeqa: reproducible: Use subTest for packages
In reply to Joshua WattRuns each package class reproducibility test in a separate sub-test. This allows the other sub tests to still run in the event that one fails.
Signed-off-by Joshua Watt <[email protected]>
--- meta/lib/oeqa/selftest/cases/reproducible.py | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-)
meta/lib/oeqa/selftest/cases/reproducible.py +11 -10
@@ -134,7 +134,7 @@ class ReproducibleTests(OESelftestTestCase): reproducible_tmp = os.path.join(self.topdir, 'reproducible', 'tmp') if os.path.exists(reproducible_tmp): - bb.utils.remove(reproducible_tmp) + bb.utils.remove(reproducible_tmp, recurse=True) # Perform another build. This build should *not* share sstate or pull # from any mirrors, but sharing a DL_DIR is fine
@@ -150,18 +150,19 @@ class ReproducibleTests(OESelftestTestCase): # kept after the build so it can be diffed for debugging. for c in self.package_classes: - package_class = 'package_' + c + with self.subTest(package_class=c): + package_class = 'package_' + c - deploy_reference = vars_reference['DEPLOY_DIR_' + c.upper()] - deploy_test = vars_test['DEPLOY_DIR_' + c.upper()] + deploy_reference = vars_reference['DEPLOY_DIR_' + c.upper()] + deploy_test = vars_test['DEPLOY_DIR_' + c.upper()] - result = self.compare_packages(deploy_reference, deploy_test, diffutils_sysroot) + result = self.compare_packages(deploy_reference, deploy_test, diffutils_sysroot) - self.logger.info('Reproducibility summary for %s: %s' % (c, result)) + self.logger.info('Reproducibility summary for %s: %s' % (c, result)) - self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) + self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) - if result.missing or result.different: - self.fail("The following %s packages are missing or different: %s" % - (c, ' '.join(r.test for r in (result.missing + result.different)))) + if result.missing or result.different: + self.fail("The following %s packages are missing or different: %s" % + (c, ' '.join(r.test for r in (result.missing + result.different))))
Signature · 1 line
2.21.0
PATCH 3/5 oeqa: Set LD_LIBRARY_PATH when executing native commands
In reply to Joshua WattSome commands like to look for libraries at runtime manually (e.g. Python's ctype.utils.find_library() function). For this to work properly, the libraries in the native sysroot must be findable. To accomplish this, set LD_LIBRARY_PATH to search library paths in the native sysroot.
Signed-off-by Joshua Watt <[email protected]>
--- meta/lib/oeqa/utils/commands.py | 3 +++ 1 file changed, 3 insertions(+)
meta/lib/oeqa/utils/commands.py +3 -0
@@ -172,8 +172,11 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, if native_sysroot: extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ (native_sysroot, native_sysroot, native_sysroot) + extra_libpaths = "%s/lib:%s/usr/lib" % \ + (native_sysroot, native_sysroot) nenv = dict(options.get('env', os.environ)) nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') + nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '') options['env'] = nenv cmd = Command(command, timeout=timeout, output_log=output_log, **options)
Signature · 1 line
2.21.0
PATCH 4/5 oeqa: reproducible: Record packages in test results
In reply to Joshua WattRecords the results of the reproducibility analysis in the JSON test results file. This makes it easier to do post-test analysis on the packages.
[YOCTO #13324]
Signed-off-by Joshua Watt <[email protected]>
--- meta/lib/oeqa/selftest/cases/reproducible.py | 10 ++++++++++ 1 file changed, 10 insertions(+)
meta/lib/oeqa/selftest/cases/reproducible.py +10 -0
@@ -8,6 +8,7 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import functools import multiprocessing import textwrap +import json import unittest MISSING = 'MISSING'
@@ -86,6 +87,7 @@ class ReproducibleTests(OESelftestTestCase): self.extras = self.tc.extraresults self.extras.setdefault('reproducible.rawlogs', {})['log'] = '' + self.extras.setdefault('reproducible', {}).setdefault('files', {}) def append_to_log(self, msg): self.extras['reproducible.rawlogs']['log'] += msg
@@ -114,6 +116,10 @@ class ReproducibleTests(OESelftestTestCase): result.sort() return result + def write_package_list(self, package_class, name, packages): + self.extras['reproducible']['files'].setdefault(package_class, {})[name] = [ + {'reference': p.reference, 'test': p.test} for p in packages] + @unittest.skip("Reproducible builds do not yet pass") def test_reproducible_builds(self): capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
@@ -162,6 +168,10 @@ class ReproducibleTests(OESelftestTestCase): self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) + self.write_package_list(package_class, 'missing', result.missing) + self.write_package_list(package_class, 'different', result.different) + self.write_package_list(package_class, 'same', result.same) + if result.missing or result.different: self.fail("The following %s packages are missing or different: %s" % (c, ' '.join(r.test for r in (result.missing + result.different))))
Signature · 1 line
2.21.0
PATCH 5/5 oeqa: Enable reproducible build test
In reply to Joshua Watt[YOCTO #13323]
Signed-off-by Joshua Watt <[email protected]>
--- meta/lib/oeqa/selftest/cases/reproducible.py | 1 - 1 file changed, 1 deletion(-)
meta/lib/oeqa/selftest/cases/reproducible.py +0 -1
@@ -120,7 +120,6 @@ class ReproducibleTests(OESelftestTestCase): self.extras['reproducible']['files'].setdefault(package_class, {})[name] = [ {'reference': p.reference, 'test': p.test} for p in packages] - @unittest.skip("Reproducible builds do not yet pass") def test_reproducible_builds(self): capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
Signature · 1 line
2.21.0