pull request
cover pull request
Please concider these changes for zeus
The following changes since commit f9739e0f58f5d8a5dc01fdb0efaa778af4edd671:
kernel: Make symbol link to vmlinux.64 in boot directory (2020-01-08 20:33:12 -0800)
are available in the Git repository at:
git://git.openembedded.org/openembedded-core-contrib stable/zeus-next
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=stable/zeus-next
Alex Kiernan (1):
linuxloader: Correct loader for glibc on armhf
Kalle Lampila (1):
wic/filemap: If FIGETBSZ iotctl fail, failback to os.stat
Mike Crowe (1):
multilib.conf: Ensure that RECIPE_SYSROOT is unchanged for native
Peter Kjellerstedt (5):
toaster.bbclass: Correct pkgdatadir path in toaster_package_dumpdata() populate_sdk_ext.bbclass: No longer needed to clean away
conf/sanity_info
sanity.bbclass: Move sanity_info from conf to cache licenses.conf: Remove the SRC_DISTRIBUTE_LICENSES variable license.bbclass: Introduce AVAILABLE_LICENSES that lists all licenses
Richard Purdie (3):
sstatesig: Test cross/native hashserv method extension scripts/oe-build-perf-report: Avoid buildstats warning
sstatesig: Improve debug output if getpwuid() fails
meta/classes/license.bbclass | 27 ++++++++++-- meta/classes/linuxloader.bbclass | 2 +- meta/classes/populate_sdk_ext.bbclass | 2 +- meta/classes/sanity.bbclass | 2 +- meta/classes/toaster.bbclass | 15 +++++-- meta/conf/documentation.conf | 1 + meta/conf/licenses.conf | 43 ------------------- meta/conf/multilib.conf | 1 + meta/lib/oe/sstatesig.py | 17 +++++++- meta/lib/oeqa/buildperf/base.py | 2 +- .../oeqa/selftest/cases/incompatible_lic.py | 6 +-- meta/lib/oeqa/selftest/cases/sstatetests.py | 40 +++++++++++++++++ scripts/lib/wic/filemap.py | 6 ++- scripts/oe-build-perf-report | 20 ++++++--- 14 files changed, 116 insertions(+), 68 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/11 toaster.bbclass: Correct pkgdatadir path in toaster_package_dumpdata()
From: Peter Kjellerstedt <[email protected]>
In commit 692b2046 (package: Fix race between do_package and do_packagedata), the path used for do_packagedata[sstate-inputdirs] was changed from "${PKGDESTWORK}" to "${WORKDIR}/pkgdata-pdata-input". This commit adapts the path used for pkgdatadir in toaster_package_dumpdata() accordingly to avoid setscene errors like:
ERROR: libgcc-9.2.0-r0 do_packagedata_setscene: Error executing a python function in exec_python_func() autogenerated:
The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_python_func() autogenerated', lineno: 2, function:
<module>
0001:
*** 0002:toaster_package_dumpdata(d)
0003: File: 'meta/classes/toaster.bbclass', lineno: 130, function:
toaster_package_dumpdata
0126: lpkgdata = {}
0127: datadir = os.path.join(pkgdatadir, 'runtime')
0128:
0129: # scan and send data for each generated package
*** 0130: for datafile in os.listdir(datadir):
0131: if not datafile.endswith('.packaged'):
0132: lpkgdata = _toaster_load_pkgdatafile(datadir,
0133: # Fire an event containing the pkg data
0134: bb.event.fire(bb.event.MetadataEvent(
Exception: FileNotFoundError: [Errno 2] No such file or directory:
'tmp/work/mips32r2el-nf-poky-linux/libgcc/9.2.0-r0/pkgdata/runtime'
--- meta/classes/toaster.bbclass | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
meta/classes/toaster.bbclass +11 -4
@@ -113,7 +113,7 @@ def _toaster_load_pkgdatafile(dirpath, filepath): pass # ignore lines without valid key: value pairs return pkgdata -python toaster_package_dumpdata() { +def _toaster_dumpdata(pkgdatadir, d): """ Dumps the data about the packages created by a recipe """
@@ -122,7 +122,6 @@ python toaster_package_dumpdata() { if not d.getVar('PACKAGES'): return - pkgdatadir = d.getVar('PKGDESTWORK') lpkgdata = {} datadir = os.path.join(pkgdatadir, 'runtime')
@@ -132,6 +131,14 @@ python toaster_package_dumpdata() { lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) # Fire an event containing the pkg data bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) + +python toaster_package_dumpdata() { + _toaster_dumpdata(d.getVar('PKGDESTWORK'), d) +} + +python toaster_packagedata_dumpdata() { + # This path needs to match do_packagedata[sstate-inputdirs] + _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d) } # 2. Dump output image files information
@@ -366,8 +373,8 @@ toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" addhandler toaster_artifacts toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted" -do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata " -do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata " +do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata " +do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata " do_package[postfuncs] += "toaster_package_dumpdata " do_package[vardepsexclude] += "toaster_package_dumpdata "
Signature · 1 line
2.17.1
PATCH 2/11 populate_sdk_ext.bbclass: No longer needed to clean away conf/sanity_info
In reply to Armin KusterFrom: Peter Kjellerstedt <[email protected]>
Since the sanity_info file has moved from the conf directory to the cache directory, there is no longer any need to clean it away explicitly in clean_esdk_builddir() since the whole cache directory is already cleaned away anyway.
--- meta/classes/populate_sdk_ext.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/populate_sdk_ext.bbclass +1 -1
@@ -122,7 +122,7 @@ SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTR def clean_esdk_builddir(d, sdkbasepath): """Clean up traces of the fake build for create_filtered_tasklist()""" import shutil - cleanpaths = 'cache conf/sanity_info tmp'.split() + cleanpaths = ['cache', 'tmp'] for pth in cleanpaths: fullpth = os.path.join(sdkbasepath, pth) if os.path.isdir(fullpth):
Signature · 1 line
2.17.1
PATCH 3/11 sanity.bbclass: Move sanity_info from conf to cache
In reply to Armin KusterFrom: Peter Kjellerstedt <[email protected]>
Since this file is written during recipe parsing, having it in the ${BUILDDIR}/conf directory, which is covered by an inotify watcher, will trigger a re-parse the next time bitbake is run and the resident bitbake server is enabled. This causes the sanity_info file to be updated again, which triggers a new parse the next time bitbake is run ad infinitum. Moving it to ${BUILDDIR}/cache should avoid this.
--- meta/classes/sanity.bbclass | 2 +- meta/lib/oeqa/buildperf/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
meta/classes/sanity.bbclass +1 -1
@@ -927,7 +927,7 @@ def check_sanity(sanity_data): last_tmpdir = "" last_sstate_dir = "" last_nativelsbstr = "" - sanityverfile = sanity_data.expand("${TOPDIR}/conf/sanity_info") + sanityverfile = sanity_data.expand("${TOPDIR}/cache/sanity_info") if os.path.exists(sanityverfile): with open(sanityverfile, 'r') as f: for line in f:
meta/lib/oeqa/buildperf/base.py +1 -1
@@ -462,7 +462,7 @@ class BuildPerfTestCase(unittest.TestCase): def rm_tmp(self): """Cleanup temporary/intermediate files and directories""" log.debug("Removing temporary and cache files") - for name in ['bitbake.lock', 'conf/sanity_info', + for name in ['bitbake.lock', 'cache/sanity_info', self.bb_vars['TMPDIR']]: oe.path.remove(name, recurse=True)
Signature · 1 line
2.17.1
PATCH 4/11 licenses.conf: Remove the SRC_DISTRIBUTE_LICENSES variable
In reply to Armin KusterFrom: Peter Kjellerstedt <[email protected]>
The SRC_DISTRIBUTE_LICENSES variable and its static list of licenses has been replaced by AVAILABLE_LICENSES, which automatically contains all available licenses.
--- meta/conf/licenses.conf | 43 ----------------------------------------- 1 file changed, 43 deletions(-)
meta/conf/licenses.conf +0 -43
@@ -1,44 +1,3 @@ -# These aren't actually used anywhere that I can tell -# They may be in the future (or are used by someone else -# For completion sake, I've updated them -SRC_DISTRIBUTE_LICENSES += "AAL Adobe AFL-1.2 AFL-2.0 AFL-2.1 AFL-3.0" -SRC_DISTRIBUTE_LICENSES += "AGPL-3.0 ANTLR-PD Apache-1.0 Apache-1.1 Apache-2.0" -SRC_DISTRIBUTE_LICENSES += "APL-1.0 APSL-1.0 APSL-1.1 APSL-1.2 APSL-2.0" -SRC_DISTRIBUTE_LICENSES += "Artistic-1.0 Artistic-2.0 BitstreamVera BSD" -SRC_DISTRIBUTE_LICENSES += "BSD-2-Clause BSD-3-Clause BSD-4-Clause BSL-1.0" -SRC_DISTRIBUTE_LICENSES += "CATOSL-1.1 CC0-1.0 CC-BY-1.0 CC-BY-2.0 CC-BY-2.5" -SRC_DISTRIBUTE_LICENSES += "CC-BY-3.0 CC-BY-NC-1.0 CC-BY-NC-2.0 CC-BY-NC-2.5" -SRC_DISTRIBUTE_LICENSES += "CC-BY-NC-3.0 CC-BY-NC-ND-1.0 CC-BY-NC-ND-2.0" -SRC_DISTRIBUTE_LICENSES += "CC-BY-NC-ND-2.5 CC-BY-NC-ND-3.0 CC-BY-NC-SA-1.0" -SRC_DISTRIBUTE_LICENSES += "CC-BY-NC-SA-2.0 CC-BY-NC-SA-2.5 CC-BY-NC-SA-3.0" -SRC_DISTRIBUTE_LICENSES += "CC-BY-ND-1.0 CC-BY-ND-2.0 CC-BY-ND-2.5 CC-BY-ND-3.0" -SRC_DISTRIBUTE_LICENSES += "CC-BY-SA-1.0 CC-BY-SA-2.0 CC-BY-SA-2.5 CC-BY-SA-3.0 CC-BY-SA-4.0" -SRC_DISTRIBUTE_LICENSES += "CDDL-1.0 CECILL-1.0 CECILL-2.0 CECILL-B CECILL-C" -SRC_DISTRIBUTE_LICENSES += "ClArtistic CPAL-1.0 CPL-1.0 CUA-OPL-1.0 DSSSL" -SRC_DISTRIBUTE_LICENSES += "ECL-1.0 ECL-2.0 eCos-2.0 EDL-1.0 EFL-1.0 EFL-2.0" -SRC_DISTRIBUTE_LICENSES += "Entessa EPL-1.0 EPL-2.0 ErlPL-1.1" -SRC_DISTRIBUTE_LICENSES += "EUDatagrid EUPL-1.0 EUPL-1.1 Fair Frameworx-1.0" -SRC_DISTRIBUTE_LICENSES += "FreeType GFDL-1.1 GFDL-1.2 GFDL-1.3 GPL-1.0" -SRC_DISTRIBUTE_LICENSES += "GPL-2.0 GPL-2.0-with-autoconf-exception" -SRC_DISTRIBUTE_LICENSES += "GPL-2.0-with-classpath-exception" -SRC_DISTRIBUTE_LICENSES += "GPL-2.0-with-font-exception" -SRC_DISTRIBUTE_LICENSES += "GPL-2.0-with-GCC-exception" -SRC_DISTRIBUTE_LICENSES += "GPL-2-with-bison-exception GPL-3.0" -SRC_DISTRIBUTE_LICENSES += "GPL-3.0-with-autoconf-exception" -SRC_DISTRIBUTE_LICENSES += "GPL-3.0-with-GCC-exception" -SRC_DISTRIBUTE_LICENSES += "gSOAP-1 gSOAP-1.3b HPND IPA IPL-1.0 ISC LGPL-2.0" -SRC_DISTRIBUTE_LICENSES += "LGPL-2.1 LGPL-3.0 Libpng LPL-1.02 LPPL-1.0 LPPL-1.1" -SRC_DISTRIBUTE_LICENSES += "LPPL-1.2 LPPL-1.3c MirOS MIT Motosoto MPL-1.0" -SRC_DISTRIBUTE_LICENSES += "MPL-1.1 MS-PL MS-RL Multics NASA-1.3 Nauman NCSA" -SRC_DISTRIBUTE_LICENSES += "NGPL Nokia NPOSL-3.0 NTP OASIS OCLC-2.0 ODbL-1.0" -SRC_DISTRIBUTE_LICENSES += "OFL-1.1 OGTSL OLDAP-2.8 OpenSSL OSL-1.0 OSL-2.0" -SRC_DISTRIBUTE_LICENSES += "OSL-3.0 PD PHP-3.0 PostgreSQL Proprietary" -SRC_DISTRIBUTE_LICENSES += "Python-2.0 QPL-1.0 RHeCos-1 RHeCos-1.1 RPL-1.5" -SRC_DISTRIBUTE_LICENSES += "RPSL-1.0 RSCPL Ruby SAX-PD SGI-1 Simple-2.0 Sleepycat" -SRC_DISTRIBUTE_LICENSES += "SPL-1.0 SugarCRM-1 SugarCRM-1.1.3 UCB VSL-1.0 W3C" -SRC_DISTRIBUTE_LICENSES += "Watcom-1.0 WXwindows XFree86-1.0 XFree86-1.1 Xnet XSL YPL-1.1" -SRC_DISTRIBUTE_LICENSES += "Zimbra-1.3 Zlib ZPL-1.1 ZPL-2.0 ZPL-2.1" - # Standards are great! Everyone has their own. In an effort to standardize licensing # names, common-licenses will use the SPDX standard license names. In order to not # break the non-standardized license names that we find in LICENSE, we'll set
@@ -188,5 +147,3 @@ FOSS_BASE_URL = "http://localhost/repo/?mod=spdx_license_once" FOSS_SERVER = "${FOSS_BASE_URL}&fullSPDXFlag=${FOSS_FULL_SPDX}&noCopyright=${FOSS_NO_COPYRIGHT}&recursiveUnpack=${FOSS_RECURSIVE_UNPACK}" FOSS_WGET_FLAGS = "-qO - --no-check-certificate --timeout=0" - -
Signature · 1 line
2.17.1
PATCH 5/11 license.bbclass: Introduce AVAILABLE_LICENSES that lists all licenses
In reply to Armin KusterFrom: Peter Kjellerstedt <[email protected]>
Previously, there was SRC_DISTRIBUTE_LICENSES, an undocumented variable that contained a static list of licenses. It was used by expand_wildcard_licenses() to expand any wildcards used in, e.g., INCOMPATIBLE_LICENSE. However, since this static list of licenses has not been kept up-to-date, many licenses were missing, with the result that if one tried to use any of those licenses with a wildcard, no licenses would be found, effectively ignoring that they should be marked as incompatible.
This introduces a new (documented) variable, AVAILABLE_LICENSES, that is automatically updated to contain all licenses found in any directories specified by ${COMMON_LICENSE_DIR} and ${LICENSE_PATH}, and uses it instead of SRC_DISTRIBUTE_LICENSES when expanding wildcards.
--- meta/classes/license.bbclass | 27 ++++++++++++++++--- meta/conf/documentation.conf | 1 + .../oeqa/selftest/cases/incompatible_lic.py | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-)
meta/classes/license.bbclass +23 -4
@@ -252,7 +252,7 @@ def canonical_license(d, license): """ Return the canonical (SPDX) form of the license if available (so GPLv3 becomes GPL-3.0), for the license named 'X+', return canonical form of - 'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+), + 'X' if available and the tailing '+' (so GPLv3+ becomes GPL-3.0+), or the passed license if there is no canonical form. """ lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
@@ -262,10 +262,29 @@ def canonical_license(d, license): lic += '+' return lic or license +def available_licenses(d): + """ + Return the available licenses by searching the directories specified by + COMMON_LICENSE_DIR and LICENSE_PATH. + """ + lic_dirs = ((d.getVar('COMMON_LICENSE_DIR') or '') + ' ' + + (d.getVar('LICENSE_PATH') or '')).split() + + licenses = [] + for lic_dir in lic_dirs: + licenses += os.listdir(lic_dir) + + licenses = sorted(licenses) + return licenses + +# Only determine the list of all available licenses once. This assumes that any +# additions to LICENSE_PATH have been done before this file is parsed. +AVAILABLE_LICENSES := "${@' '.join(available_licenses(d))}" + def expand_wildcard_licenses(d, wildcard_licenses): """ - Return actual spdx format license names if wildcard used. We expand - wildcards from SPDXLICENSEMAP flags and SRC_DISTRIBUTE_LICENSES values. + Return actual spdx format license names if wildcards are used. We expand + wildcards from SPDXLICENSEMAP flags and AVAILABLE_LICENSES. """ import fnmatch licenses = wildcard_licenses[:]
@@ -274,7 +293,7 @@ def expand_wildcard_licenses(d, wildcard_licenses): spdxflags = fnmatch.filter(spdxmapkeys, wld_lic) licenses += [d.getVarFlag('SPDXLICENSEMAP', flag) for flag in spdxflags] - spdx_lics = (d.getVar('SRC_DISTRIBUTE_LICENSES', False) or '').split() + spdx_lics = d.getVar('AVAILABLE_LICENSES').split() for wld_lic in wildcard_licenses: licenses += fnmatch.filter(spdx_lics, wld_lic)
meta/conf/documentation.conf +1 -0
@@ -69,6 +69,7 @@ ASSUME_SHLIBS[doc] = "List of shlib:package[_version] mappings. Useful for lib p AUTHOR[doc] = "Email address used to contact the original author(s) in order to send patches and forward bugs." AUTO_SYSLINUXMENU[doc] = "Enables creating an automatic menu for the syslinux bootloader." AUTOREV[doc] = "When SRCREV is set to the value of this variable, it specifies to use the latest source revision in the repository." +AVAILABLE_LICENSES[doc] = "List of licenses found in the directories specified by COMMON_LICENSE_DIR and LICENSE_PATH." #B
meta/lib/oeqa/selftest/cases/incompatible_lic.py +3 -3
@@ -12,12 +12,12 @@ class IncompatibleLicenseTests(OESelftestTestCase): if error_msg not in result.output: raise AssertionError(result.output) - # Verify that a package with an SPDX license (from SRC_DISTRIBUTE_LICENSES) + # Verify that a package with an SPDX license (from AVAILABLE_LICENSES) # cannot be built when INCOMPATIBLE_LICENSE contains this SPDX license def test_incompatible_spdx_license(self): self.lic_test('incompatible-license', 'GPL-3.0', 'GPL-3.0') - # Verify that a package with an SPDX license (from SRC_DISTRIBUTE_LICENSES) + # Verify that a package with an SPDX license (from AVAILABLE_LICENSES) # cannot be built when INCOMPATIBLE_LICENSE contains an alias (in # SPDXLICENSEMAP) of this SPDX license def test_incompatible_alias_spdx_license(self):
@@ -35,7 +35,7 @@ class IncompatibleLicenseTests(OESelftestTestCase): self.lic_test('incompatible-license-alias', 'GPLv3', 'GPLv3') # Verify that a package with a non-SPDX license (neither in - # SRC_DISTRIBUTE_LICENSES nor in SPDXLICENSEMAP) cannot be built when + # AVAILABLE_LICENSES nor in SPDXLICENSEMAP) cannot be built when # INCOMPATIBLE_LICENSE contains this license def test_incompatible_nonspdx_license(self): self.lic_test('incompatible-nonspdx-license', 'FooLicense', 'FooLicense')
Signature · 1 line
2.17.1
PATCH 6/11 linuxloader: Correct loader for glibc on armhf
In reply to Armin KusterFrom: Alex Kiernan <[email protected]>
Fix the naming of the loader on armhf to match what glibc expects:
#define GLIBC_DYNAMIC_LINKER_SOFT_FLOAT SYSTEMLIBS_DIR "ld-linux.so.3" #define GLIBC_DYNAMIC_LINKER_HARD_FLOAT SYSTEMLIBS_DIR "ld-linux-armhf.so.3"
--- meta/classes/linuxloader.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/linuxloader.bbclass +1 -1
@@ -41,7 +41,7 @@ def get_glibc_loader(d): elif re.search("i.86", targetarch): dynamic_loader = "${base_libdir}/ld-linux.so.2" elif targetarch == "arm": - dynamic_loader = "${base_libdir}/ld-linux.so.3" + dynamic_loader = "${base_libdir}/ld-linux${@['-armhf', ''][d.getVar('TARGET_FPU') == 'soft']}.so.3" elif targetarch.startswith("aarch64"): dynamic_loader = "${base_libdir}/ld-linux-aarch64${ARMPKGSFX_ENDIAN_64}.so.1" elif targetarch.startswith("riscv64"):
Signature · 1 line
2.17.1
PATCH 7/11 multilib.conf: Ensure that RECIPE_SYSROOT is unchanged for native
In reply to Armin KusterFrom: Mike Crowe <[email protected]>
Ensure that RECIPE_SYSROOT is the same for -native recipes whether multilib.conf is included or not.
Without this change task signatures for -native recipes change when switching between MACHINEs that require multilib.conf and those that don't.
This fix was one of the ones suggested by Khem Raj in
http://lists.openembedded.org/pipermail/openembedded-core/2019-December/290303.html
Add test_sstate_multilib_or_not_native_samesigs test case to sstatetests.py to ensure that this stays fixed.
--- meta/conf/multilib.conf | 1 + meta/lib/oeqa/selftest/cases/sstatetests.py | 40 +++++++++++++++++++++ 2 files changed, 41 insertions(+)
meta/conf/multilib.conf +1 -0
@@ -9,6 +9,7 @@ MULTILIBS ??= "multilib:lib32" STAGING_DIR_HOST = "${WORKDIR}/${MLPREFIX}recipe-sysroot" STAGING_DIR_TARGET = "${WORKDIR}/${MLPREFIX}recipe-sysroot" RECIPE_SYSROOT = "${WORKDIR}/${MLPREFIX}recipe-sysroot" +RECIPE_SYSROOT_class-native = "${WORKDIR}/recipe-sysroot" INHERIT += "multilib_global"
meta/lib/oeqa/selftest/cases/sstatetests.py +40 -0
@@ -446,6 +446,46 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" self.assertCountEqual(files1, files2) + def test_sstate_multilib_or_not_native_samesigs(self): + """The sstate checksums of two native recipes (and their dependencies) + where the target is using multilib in one but not the other + should be the same. We use the qemux86copy machine to test + this. + """ + + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +TCLIBCAPPEND = \"\" +MACHINE = \"qemux86\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +BB_SIGNATURE_HANDLER = "OEBasicHash" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("binutils-native -S none") + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +TCLIBCAPPEND = \"\" +MACHINE = \"qemux86copy\" +BB_SIGNATURE_HANDLER = "OEBasicHash" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("binutils-native -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + for name in files: + f.append(os.path.join(root, name)) + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertCountEqual(files1, files2) + + def test_sstate_noop_samesigs(self): """ The sstate checksums of two builds with these variables changed or
Signature · 1 line
2.17.1
PATCH 8/11 wic/filemap: If FIGETBSZ iotctl fail, failback to os.stat
In reply to Armin KusterFrom: Kalle Lampila <[email protected]>
Some file systems don't support fetching the block size (notably the file system Docker uses for containers), so if iotctl() fail, try to use failback via os.stat() to get block size.
--- scripts/lib/wic/filemap.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
scripts/lib/wic/filemap.py +4 -2
@@ -34,9 +34,11 @@ def get_block_size(file_obj): # the FIGETBSZ ioctl (number 2). try: binary_data = fcntl.ioctl(file_obj, 2, struct.pack('I', 0)) + bsize = struct.unpack('I', binary_data)[0] except OSError: - raise IOError("Unable to determine block size") - bsize = struct.unpack('I', binary_data)[0] + bsize = None + + # If ioctl causes OSError or give bsize to zero failback to os.fstat if not bsize: import os stat = os.fstat(file_obj.fileno())
Signature · 1 line
2.17.1
PATCH 9/11 sstatesig: Test cross/native hashserv method extension
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
We can have one taskhash which represents multiple native/cross sstate objects since they're stored by BUILD_ARCH or possibly host distro (or host gcc version). We need to put these into separate namespaces on hashserv since their outhashes will never match and we need deterministic lookups for the different namespaces.
Use this extramethod option to handle this. This fixes several problematic failures on the autobuilder.
--- meta/lib/oe/sstatesig.py | 9 +++++++++ 1 file changed, 9 insertions(+)
meta/lib/oe/sstatesig.py +9 -0
@@ -103,6 +103,7 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or "").split() self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } + self.buildarch = data.getVar('BUILD_ARCH') pass def tasks_resolved(self, virtmap, virtpnmap, dataCache):
@@ -140,6 +141,14 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) + def prep_taskhash(self, tid, deps, dataCache): + super().prep_taskhash(tid, deps, dataCache) + if hasattr(self, "extramethod"): + (_, _, _, fn) = bb.runqueue.split_tid_mcfn(tid) + inherits = " ".join(dataCache.inherits[fn]) + if inherits.find("/native.bbclass") != -1 or inherits.find("/cross.bbclass") != -1: + self.extramethod[tid] = ":" + self.buildarch + def get_taskhash(self, tid, deps, dataCache): h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(tid, deps, dataCache)
Signature · 1 line
2.17.1
PATCH 10/11 scripts/oe-build-perf-report: Avoid buildstats warning
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
When a comparision branch is set we see warnings. If two branches are used, look for buildstats information from both to avoid this kind of warning:
INFO: Parsing buildstats from 'refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86'
WARNING: Buildstats not found for perf-centos7.yoctoproject.org/master/qemux86/56808-g08220028e52992dcba667fc97bf3afe8be1949fb/0
WARNING: Buildstats not found for perf-centos7.yoctoproject.org/master/qemux86/56808-g08220028e52992dcba667fc97bf3afe8be1949fb/1
INFO: Buildstats were missing for some test runs, please run 'git fetch origin refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86:refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86' and try again
(the warning is due to stats being looked for in both branches, choose the other branch and different warnings would be shown)
--- scripts/oe-build-perf-report | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)
scripts/oe-build-perf-report +14 -6
@@ -372,7 +372,7 @@ def print_html_report(data, id_comp, buildstats): chart_opts=chart_opts)) -def get_buildstats(repo, notes_ref, revs, outdir=None): +def get_buildstats(repo, notes_ref, notes_ref2, revs, outdir=None): """Get the buildstats from git notes""" full_ref = 'refs/notes/' + notes_ref if not repo.rev_parse(full_ref):
@@ -391,8 +391,13 @@ def get_buildstats(repo, notes_ref, revs, outdir=None): for tag in rev.tags: log.debug(' %s', tag) try: - bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, - 'show', tag + '^0'])) + try: + bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, 'show', tag + '^0'])) + except GitError: + if notes_ref2: + bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref2, 'show', tag + '^0'])) + else: + raise except GitError: log.warning("Buildstats not found for %s", tag) bs_all = {}
@@ -589,9 +594,12 @@ def main(argv=None): buildstats = None if args.dump_buildstats or args.html: outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None - notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, - args.machine) - buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir) + notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine) + notes_ref2 = None + if args.branch2: + notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch2, args.machine) + notes_ref2 = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine) + buildstats = get_buildstats(repo, notes_ref, notes_ref2, [rev_l, rev_r], outdir) # Print report if not args.html:
Signature · 1 line
2.17.1
PATCH 11/11 sstatesig: Improve debug output if getpwuid() fails
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
If getpwduid fails, we don't see which file it failed on which is key information to aid debugging. Print this information when exceptions are raised.
--- meta/lib/oe/sstatesig.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
meta/lib/oe/sstatesig.py +6 -2
@@ -521,8 +521,12 @@ def OEOuthashBasic(path, sigfile, task, d): add_perm(stat.S_IXOTH, 'x') if include_owners: - update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) - update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) + try: + update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) + update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) + except KeyError: + bb.warn("KeyError in %s" % path) + raise update_hash(" ") if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
Signature · 1 line
2.17.1