Patch review
cover Patch review
Please review and provide feedback by Tuesday.
Passed on AB via A-full
The following changes since commit 5ed714139f91eb03871e01b68a4370784071234d:
license.bbclass: Introduce AVAILABLE_LICENSES that lists all licenses (2020-01-15 11:58:08 +0800)
are available in the Git repository at:
git://git.openembedded.org/openembedded-core-contrib stable/zeus-nut
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=stable/zeus-nut
Alex Kiernan (1):
linuxloader: Correct loader for glibc on armhf
Kalle Lampila (1):
wic/filemap: If FIGETBSZ iotctl fail, failback to os.stat
Mike Crowe (1):
multilib.conf: Ensure that RECIPE_SYSROOT is unchanged for native
Richard Purdie (3):
sstatesig: Test cross/native hashserv method extension scripts/oe-build-perf-report: Avoid buildstats warning
sstatesig: Improve debug output if getpwuid() fails
meta/classes/linuxloader.bbclass | 2 +- meta/conf/multilib.conf | 1 + meta/lib/oe/sstatesig.py | 17 +++++++-- meta/lib/oeqa/selftest/cases/sstatetests.py | 40 +++++++++++++++++++++ scripts/lib/wic/filemap.py | 6 ++-- scripts/oe-build-perf-report | 20 +++++++---- 6 files changed, 75 insertions(+), 11 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/6 linuxloader: Correct loader for glibc on armhf
From: Alex Kiernan <[email protected]>
Fix the naming of the loader on armhf to match what glibc expects:
#define GLIBC_DYNAMIC_LINKER_SOFT_FLOAT SYSTEMLIBS_DIR "ld-linux.so.3" #define GLIBC_DYNAMIC_LINKER_HARD_FLOAT SYSTEMLIBS_DIR "ld-linux-armhf.so.3"
--- meta/classes/linuxloader.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/linuxloader.bbclass +1 -1
@@ -41,7 +41,7 @@ def get_glibc_loader(d): elif re.search("i.86", targetarch): dynamic_loader = "${base_libdir}/ld-linux.so.2" elif targetarch == "arm": - dynamic_loader = "${base_libdir}/ld-linux.so.3" + dynamic_loader = "${base_libdir}/ld-linux${@['-armhf', ''][d.getVar('TARGET_FPU') == 'soft']}.so.3" elif targetarch.startswith("aarch64"): dynamic_loader = "${base_libdir}/ld-linux-aarch64${ARMPKGSFX_ENDIAN_64}.so.1" elif targetarch.startswith("riscv64"):
Signature · 1 line
2.17.1
PATCH 2/6 multilib.conf: Ensure that RECIPE_SYSROOT is unchanged for native
In reply to Armin KusterFrom: Mike Crowe <[email protected]>
Ensure that RECIPE_SYSROOT is the same for -native recipes whether multilib.conf is included or not.
Without this change task signatures for -native recipes change when switching between MACHINEs that require multilib.conf and those that don't.
This fix was one of the ones suggested by Khem Raj in
http://lists.openembedded.org/pipermail/openembedded-core/2019-December/290303.html
Add test_sstate_multilib_or_not_native_samesigs test case to sstatetests.py to ensure that this stays fixed.
--- meta/conf/multilib.conf | 1 + meta/lib/oeqa/selftest/cases/sstatetests.py | 40 +++++++++++++++++++++ 2 files changed, 41 insertions(+)
meta/conf/multilib.conf +1 -0
@@ -9,6 +9,7 @@ MULTILIBS ??= "multilib:lib32" STAGING_DIR_HOST = "${WORKDIR}/${MLPREFIX}recipe-sysroot" STAGING_DIR_TARGET = "${WORKDIR}/${MLPREFIX}recipe-sysroot" RECIPE_SYSROOT = "${WORKDIR}/${MLPREFIX}recipe-sysroot" +RECIPE_SYSROOT_class-native = "${WORKDIR}/recipe-sysroot" INHERIT += "multilib_global"
meta/lib/oeqa/selftest/cases/sstatetests.py +40 -0
@@ -446,6 +446,46 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" self.assertCountEqual(files1, files2) + def test_sstate_multilib_or_not_native_samesigs(self): + """The sstate checksums of two native recipes (and their dependencies) + where the target is using multilib in one but not the other + should be the same. We use the qemux86copy machine to test + this. + """ + + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +TCLIBCAPPEND = \"\" +MACHINE = \"qemux86\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +BB_SIGNATURE_HANDLER = "OEBasicHash" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") + bitbake("binutils-native -S none") + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +TCLIBCAPPEND = \"\" +MACHINE = \"qemux86copy\" +BB_SIGNATURE_HANDLER = "OEBasicHash" +""") + self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") + bitbake("binutils-native -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + for name in files: + f.append(os.path.join(root, name)) + return f + files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") + files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertCountEqual(files1, files2) + + def test_sstate_noop_samesigs(self): """ The sstate checksums of two builds with these variables changed or
Signature · 1 line
2.17.1
PATCH 3/6 wic/filemap: If FIGETBSZ iotctl fail, failback to os.stat
In reply to Armin KusterFrom: Kalle Lampila <[email protected]>
Some file systems don't support fetching the block size (notably the file system Docker uses for containers), so if iotctl() fail, try to use failback via os.stat() to get block size.
--- scripts/lib/wic/filemap.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
scripts/lib/wic/filemap.py +4 -2
@@ -34,9 +34,11 @@ def get_block_size(file_obj): # the FIGETBSZ ioctl (number 2). try: binary_data = fcntl.ioctl(file_obj, 2, struct.pack('I', 0)) + bsize = struct.unpack('I', binary_data)[0] except OSError: - raise IOError("Unable to determine block size") - bsize = struct.unpack('I', binary_data)[0] + bsize = None + + # If ioctl causes OSError or give bsize to zero failback to os.fstat if not bsize: import os stat = os.fstat(file_obj.fileno())
Signature · 1 line
2.17.1
PATCH 4/6 sstatesig: Test cross/native hashserv method extension
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
We can have one taskhash which represents multiple native/cross sstate objects since they're stored by BUILD_ARCH or possibly host distro (or host gcc version). We need to put these into separate namespaces on hashserv since their outhashes will never match and we need deterministic lookups for the different namespaces.
Use this extramethod option to handle this. This fixes several problematic failures on the autobuilder.
--- meta/lib/oe/sstatesig.py | 9 +++++++++ 1 file changed, 9 insertions(+)
meta/lib/oe/sstatesig.py +9 -0
@@ -103,6 +103,7 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or "").split() self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } + self.buildarch = data.getVar('BUILD_ARCH') pass def tasks_resolved(self, virtmap, virtpnmap, dataCache):
@@ -140,6 +141,14 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) + def prep_taskhash(self, tid, deps, dataCache): + super().prep_taskhash(tid, deps, dataCache) + if hasattr(self, "extramethod"): + (_, _, _, fn) = bb.runqueue.split_tid_mcfn(tid) + inherits = " ".join(dataCache.inherits[fn]) + if inherits.find("/native.bbclass") != -1 or inherits.find("/cross.bbclass") != -1: + self.extramethod[tid] = ":" + self.buildarch + def get_taskhash(self, tid, deps, dataCache): h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(tid, deps, dataCache)
Signature · 1 line
2.17.1
PATCH 5/6 scripts/oe-build-perf-report: Avoid buildstats warning
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
When a comparision branch is set we see warnings. If two branches are used, look for buildstats information from both to avoid this kind of warning:
INFO: Parsing buildstats from 'refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86'
WARNING: Buildstats not found for perf-centos7.yoctoproject.org/master/qemux86/56808-g08220028e52992dcba667fc97bf3afe8be1949fb/0
WARNING: Buildstats not found for perf-centos7.yoctoproject.org/master/qemux86/56808-g08220028e52992dcba667fc97bf3afe8be1949fb/1
INFO: Buildstats were missing for some test runs, please run 'git fetch origin refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86:refs/notes/buildstats/perf-centos7.yoctoproject.org/master-next/qemux86' and try again
(the warning is due to stats being looked for in both branches, choose the other branch and different warnings would be shown)
--- scripts/oe-build-perf-report | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)
scripts/oe-build-perf-report +14 -6
@@ -372,7 +372,7 @@ def print_html_report(data, id_comp, buildstats): chart_opts=chart_opts)) -def get_buildstats(repo, notes_ref, revs, outdir=None): +def get_buildstats(repo, notes_ref, notes_ref2, revs, outdir=None): """Get the buildstats from git notes""" full_ref = 'refs/notes/' + notes_ref if not repo.rev_parse(full_ref):
@@ -391,8 +391,13 @@ def get_buildstats(repo, notes_ref, revs, outdir=None): for tag in rev.tags: log.debug(' %s', tag) try: - bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, - 'show', tag + '^0'])) + try: + bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, 'show', tag + '^0'])) + except GitError: + if notes_ref2: + bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref2, 'show', tag + '^0'])) + else: + raise except GitError: log.warning("Buildstats not found for %s", tag) bs_all = {}
@@ -589,9 +594,12 @@ def main(argv=None): buildstats = None if args.dump_buildstats or args.html: outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None - notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, - args.machine) - buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir) + notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine) + notes_ref2 = None + if args.branch2: + notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch2, args.machine) + notes_ref2 = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, args.machine) + buildstats = get_buildstats(repo, notes_ref, notes_ref2, [rev_l, rev_r], outdir) # Print report if not args.html:
Signature · 1 line
2.17.1
PATCH 6/6 sstatesig: Improve debug output if getpwuid() fails
In reply to Armin KusterFrom: Richard Purdie <[email protected]>
If getpwduid fails, we don't see which file it failed on which is key information to aid debugging. Print this information when exceptions are raised.
--- meta/lib/oe/sstatesig.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
meta/lib/oe/sstatesig.py +6 -2
@@ -521,8 +521,12 @@ def OEOuthashBasic(path, sigfile, task, d): add_perm(stat.S_IXOTH, 'x') if include_owners: - update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) - update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) + try: + update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) + update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) + except KeyError: + bb.warn("KeyError in %s" % path) + raise update_hash(" ") if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
Signature · 1 line
2.17.1