bblayers/makesetup: raise exceptions when errors happen
PATCH 1/9 bblayers/makesetup: raise exceptions when errors happen
From: Alexander Kanavin <[email protected]>
Otherwise the calling code can only issue a generic, unhelpful erorr message, and it's difficult to tell what went wrong if logger.error output is obscured or redirected.
--- meta/lib/bblayers/makesetup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
meta/lib/bblayers/makesetup.py +6 -4
@@ -48,8 +48,9 @@ class MakeSetupPlugin(LayerPlugin): if l_name == 'workspace': continue if l_ismodified: - logger.error("Layer {name} in {path} has uncommitted modifications or is not in a git repository.".format(name=l_name,path=l_path)) - return + e = "Layer {name} in {path} has uncommitted modifications or is not in a git repository.".format(name=l_name,path=l_path) + logger.error(e) + raise Exception(e) repo_path = oe.buildcfg.get_metadata_git_toplevel(l_path) if self._is_submodule(repo_path):
@@ -63,8 +64,9 @@ class MakeSetupPlugin(LayerPlugin): if repo_path == destdir_repo: repos[repo_path]['contains_this_file'] = True if not repos[repo_path]['git-remote']['remotes'] and not repos[repo_path]['contains_this_file']: - logger.error("Layer repository in {path} does not have any remotes configured. Please add at least one with 'git remote add'.".format(path=repo_path)) - return + e = "Layer repository in {path} does not have any remotes configured. Please add at least one with 'git remote add'.".format(path=repo_path) + logger.error(e) + raise Exception(e) top_path = os.path.commonpath([os.path.dirname(r) for r in repos.keys()])
Signature · 1 line
2.39.2
PATCH 2/9 bblayers/makesetup: include local repository paths in discovered layer repo data
From: Alexander Kanavin <[email protected]>
This can be useful when the caller needs to know where the repos actually are on local disk (for example to make a local copy of them or obtain additional information about them).
This field however should be removed from the json output in oe-setup-layers plugin, as it is host-specific and not part of the schema.
--- meta/lib/bblayers/makesetup.py | 1 + meta/lib/bblayers/setupwriters/oe-setup-layers.py | 2 ++ 2 files changed, 3 insertions(+)
meta/lib/bblayers/makesetup.py +1 -0
@@ -76,6 +76,7 @@ class MakeSetupPlugin(LayerPlugin): repos_nopaths[r_nopath] = repos[r] r_relpath = os.path.relpath(r, top_path) repos_nopaths[r_nopath]['path'] = r_relpath + repos_nopaths[r_nopath]['originpath'] = r return repos_nopaths def do_make_setup(self, args):
meta/lib/bblayers/setupwriters/oe-setup-layers.py +2 -0
@@ -85,6 +85,8 @@ class OeSetupLayersWriter(): if not os.path.exists(args.destdir): os.makedirs(args.destdir) repos = parent.make_repo_config(args.destdir) + for r in repos.values(): + del r['originpath'] json = {"version":"1.0","sources":repos} if not repos: err = "Could not determine layer sources"
Signature · 1 line
2.39.2
PATCH 3/9 bblayers/makesetup: move check for existence of git remotes to oe-setup-layers plugin
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This helps when making a local copy of all the layers: it's fine to not have any remotes configured in that case.
Also correct the mistake in accessing 'contains_this_file' property:
it may not necessarily exist, and if so, it should default to False.
--- meta/lib/bblayers/makesetup.py | 4 ---- meta/lib/bblayers/setupwriters/oe-setup-layers.py | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-)
meta/lib/bblayers/makesetup.py +0 -4
@@ -63,10 +63,6 @@ class MakeSetupPlugin(LayerPlugin): 'describe':oe.buildcfg.get_metadata_git_describe(repo_path)}} if repo_path == destdir_repo: repos[repo_path]['contains_this_file'] = True - if not repos[repo_path]['git-remote']['remotes'] and not repos[repo_path]['contains_this_file']: - e = "Layer repository in {path} does not have any remotes configured. Please add at least one with 'git remote add'.".format(path=repo_path) - logger.error(e) - raise Exception(e) top_path = os.path.commonpath([os.path.dirname(r) for r in repos.keys()])
meta/lib/bblayers/setupwriters/oe-setup-layers.py +3 -0
@@ -86,6 +86,9 @@ class OeSetupLayersWriter(): os.makedirs(args.destdir) repos = parent.make_repo_config(args.destdir) for r in repos.values(): + if not r['git-remote']['remotes'] and not r.get('contains_this_file', False): + e = "Layer repository in {path} does not have any remotes configured. Please add at least one with 'git remote add'.".format(path=r['originpath']) + raise Exception(e) del r['originpath'] json = {"version":"1.0","sources":repos} if not repos:
Signature · 1 line
2.39.2
PATCH 4/9 bblayers/setupwriters/oe-local-copy: add a 'local copy' plugin for bitbake-layers create-layer-setup
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This plugin copies all currently configured layer respositories into a dedicated location on local disk. This is useful for entirely offline layer replication, when the layers are packed and then unpacked from an archive, rather than fetched from git (there can be situations where fetching from git is undesirable or impossible).
This plugin will also be used when replicating an entire yocto build (with build config and sstate cache), and is an element of that.
This plugin is similar to what esdk does, and it provides that functionality outside of populate_sdk_ext task. It does not reuse esdk code, as that simply copies the layer tree; it's better to use 'git clone' with a path to original repo on local disk, as that will preserve commit history in the copy.
--- .../bblayers/setupwriters/oe-local-copy.py | 41 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/bblayers.py | 13 ++++++ 2 files changed, 54 insertions(+) create mode 100644 meta/lib/bblayers/setupwriters/oe-local-copy.py
meta/lib/bblayers/setupwriters/oe-local-copy.py new file +41 -0
@@ -0,0 +1,41 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import logging +import json + +logger = logging.getLogger('bitbake-layers') + +def plugin_init(plugins): + return OeLocalCopyWriter() + +class OeLocalCopyWriter(): + + def __str__(self): + return "oe-local-copy" + + def do_write(self, parent, args): + """ Writes out a local copy of all the metadata layers (and bitbake) included in a current build. """ + if not os.path.exists(args.destdir): + os.makedirs(args.destdir) + repos = parent.make_repo_config(args.destdir) + if not repos: + raise Exception("Could not determine layer sources") + output = os.path.join(os.path.abspath(args.destdir), args.output_prefix or "layers") + json_f = os.path.join(os.path.abspath(args.destdir), "bundle-repos.json") + + for r in repos.values(): + r['git-remote']['remotes'] = {"origin":{"uri":r["originpath"]}} + + json_data = {"version":"1.0","sources":repos} + with open(json_f, 'w') as f: + json.dump(json_data, f, sort_keys=True, indent=4) + + logger.info("Cloning repositories into {}".format(output)) + bb.process.run('oe-setup-layers --force-bootstraplayer-checkout --destdir {} --jsondata {}'.format(output, json_f)) + + def register_arguments(self, parser): + pass
meta/lib/oeqa/selftest/cases/bblayers.py +13 -0
@@ -240,3 +240,16 @@ class BitbakeLayers(OESelftestTestCase): self.assertEqual(first_desc_2, '', "Describe not cleared: '{}'".format(first_desc_2)) self.assertEqual(second_rev_2, second_rev_1, "Revision should not be updated: '{}'".format(second_rev_2)) self.assertEqual(second_desc_2, second_desc_1, "Describe should not be updated: '{}'".format(second_desc_2)) + + def test_bitbakelayers_setup_localcopy(self): + testcopydir = os.path.join(self.builddir, 'test-layer-copy') + result = runCmd('bitbake-layers create-layers-setup --writer oe-local-copy {}'.format(testcopydir)) + oe_core_found = False + meta_selftest_found = False + for topdir, subdirs, files in os.walk(testcopydir): + if topdir.endswith('meta/conf') and 'layer.conf' in files: + oe_core_found = True + if topdir.endswith('meta-selftest/conf') and 'layer.conf' in files: + meta_selftest_found = True + self.assertTrue(oe_core_found, "meta/conf/layer.conf not found in {}".format(testcopydir)) + self.assertTrue(meta_selftest_found, "meta-selftest/conf/layer.conf not found in {}".format(testcopydir))
Signature · 1 line
2.39.2
PATCH 5/9 lib/oe/sstatesig.py: add an option to make a unihash cache copy
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This is already performed in eSDK creation; the change allows also making a copy with a standalone 'bitbake -S unihash-cache-copy targets...' command, which is useful for replicating builds on a different build host (simply coping sstate objects is not enough if they need to be discovered through unihash redirections).
--- meta/lib/oe/sstatesig.py | 4 ++++ 1 file changed, 4 insertions(+)
meta/lib/oe/sstatesig.py +4 -0
@@ -147,6 +147,10 @@ class SignatureGeneratorOEBasicHashMixIn(object): sigfile = os.getcwd() + "/locked-sigs.inc" bb.plain("Writing locked sigs to %s" % sigfile) self.dump_lockedsigs(sigfile) + if 'unihash-cache-copy' in options: + cachefile = os.getcwd() + "/bb_unihashes.dat" + bb.plain("Writing unihash cache to %s" % cachefile) + bb.parse.siggen.copy_unitaskhashes(os.getcwd()) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
Signature · 1 line
2.39.2
Re: [OE-core] [PATCH 5/9] lib/oe/sstatesig.py: add an option to make a unihash cache copy
Quoted 19 lines · Alexander Kanavin via lists.openembedded.org
From: Alexander Kanavin <[email protected]>
This is already performed in eSDK creation; the change allows also making a copy with a standalone 'bitbake -S unihash-cache-copy targets...' command, which is useful for replicating builds on a different build host (simply coping sstate objects is not enough if they need to be discovered through unihash redirections).
Signed-off-by Alexander Kanavin <[email protected]>---
meta/lib/oe/sstatesig.py | 4 ++++1 file changed, 4 insertions(+)meta/lib/oe/sstatesig.py +0
-0@@ -147,6 +147,10 @@ class SignatureGeneratorOEBasicHashMixIn(object):sigfile =
os.getcwd()+ "/locked-sigs.inc"
Quoted 8 lines
bb.plain("Writing locked sigs to %s" % sigfile)self.dump_lockedsigs(sigfile)
+ if 'unihash-cache-copy' in options:
+ cachefile =os.getcwd()+ "/bb_unihashes.dat"
+bb.plain("Writing unihash cache to %s" % cachefile)
+ bb.parse.siggen.copy_unitaskhashes(os.getcwd())
returnsuper(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
I think we can drop the piece above now after this:
https://git.yoctoproject.org/poky/commit/?id=652e3028d9763aef318ff4587f2c21a8b280003f
The copy_unitaskhashes code is really problematic within bitbake and was a horrible hack/workaround anyway so it needs to be removed.
Cheers,
Richard
PATCH 6/9 sstatesig/dump_sigs: allow setting output directory explicitly from command line option
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
The default (current directory) is unfortunately non-deterministic, as the operation is performed from bitbake-server and not from bitbake process directly. If the server was started in a different directory it will write the files there, and not where bitbake command requesting those files was started.
--- meta/lib/oe/sstatesig.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
meta/lib/oe/sstatesig.py +8 -3
@@ -143,14 +143,19 @@ class SignatureGeneratorOEBasicHashMixIn(object): super().set_taskdata(data[3:]) def dump_sigs(self, dataCache, options): + outdir = os.getcwd() + for o in options: + if o.startswith('outdir'): + outdir = o.split('=')[1] + if 'lockedsigs' in options: - sigfile = os.getcwd() + "/locked-sigs.inc" + sigfile = outdir + "/locked-sigs.inc" bb.plain("Writing locked sigs to %s" % sigfile) self.dump_lockedsigs(sigfile) if 'unihash-cache-copy' in options: - cachefile = os.getcwd() + "/bb_unihashes.dat" + cachefile = outdir + "/bb_unihashes.dat" bb.plain("Writing unihash cache to %s" % cachefile) - bb.parse.siggen.copy_unitaskhashes(os.getcwd()) + bb.parse.siggen.copy_unitaskhashes(outdir) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
Signature · 1 line
2.39.2
PATCH 7/9 selftest/sstatetests: separate sstate presence check into its own class
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This allows reusing the check function in tests other than CDN/local mirror ones, such as sstate bundle testing.
--- meta/lib/oeqa/selftest/cases/sstatetests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
meta/lib/oeqa/selftest/cases/sstatetests.py +3 -2
@@ -917,8 +917,7 @@ INHERIT += "base-do-configure-modified" """, expected_sametmp_output, expected_difftmp_output) -@OETestTag("yocto-mirrors") -class SStateMirrors(SStateBase): +class SStateCheckObjectPresence(SStateBase): def check_bb_output(self, output, exceptions, check_cdn): def is_exception(object, exceptions): for e in exceptions:
@@ -960,6 +959,8 @@ class SStateMirrors(SStateBase): self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) +@OETestTag("yocto-mirrors") +class SStateMirrors(SStateCheckObjectPresence): def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): # sstate is checked for existence of these, but they never get written out to begin with exceptions += ["{}.*image_qa".format(t) for t in targets.split()]
Signature · 1 line
2.39.2
PATCH 8/9 selftest/sstatetests: move exception list for cache objects to the parent class
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This is useful not just in cdn/local cache tests, but also in the build bundle test, and so should be applied by default.
--- meta/lib/oeqa/selftest/cases/sstatetests.py | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-)
meta/lib/oeqa/selftest/cases/sstatetests.py +13 -13
@@ -918,13 +918,24 @@ INHERIT += "base-do-configure-modified" expected_sametmp_output, expected_difftmp_output) class SStateCheckObjectPresence(SStateBase): - def check_bb_output(self, output, exceptions, check_cdn): + def check_bb_output(self, output, targets, exceptions, check_cdn): def is_exception(object, exceptions): for e in exceptions: if re.search(e, object): return True return False + # sstate is checked for existence of these, but they never get written out to begin with + exceptions += ["{}.*image_qa".format(t) for t in targets.split()] + exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] + exceptions += ["{}.*image_complete".format(t) for t in targets.split()] + exceptions += ["linux-yocto.*shared_workdir"] + # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) + # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, + # which makes tracing other changes difficult + exceptions += ["{}.*create_spdx".format(t) for t in targets.split()] + exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()] + output_l = output.splitlines() for l in output_l: if l.startswith("Sstate summary"):
@@ -962,17 +973,6 @@ class SStateCheckObjectPresence(SStateBase): @OETestTag("yocto-mirrors") class SStateMirrors(SStateCheckObjectPresence): def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): - # sstate is checked for existence of these, but they never get written out to begin with - exceptions += ["{}.*image_qa".format(t) for t in targets.split()] - exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] - exceptions += ["{}.*image_complete".format(t) for t in targets.split()] - exceptions += ["linux-yocto.*shared_workdir"] - # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) - # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, - # which makes tracing other changes difficult - exceptions += ["{}.*create_spdx".format(t) for t in targets.split()] - exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()] - if check_cdn: self.config_sstate(True) self.append_config("""
@@ -988,7 +988,7 @@ MACHINE = "{}" bitbake("-S none {}".format(targets)) if ignore_errors: return - self.check_bb_output(result.output, exceptions, check_cdn) + self.check_bb_output(result.output, targets, exceptions, check_cdn) def test_cdn_mirror_qemux86_64(self): exceptions = []
Signature · 1 line
2.39.2
PATCH 9/9 scripts/oe-replicate-build: add a script that packages and replicates a yocto build elsewhere
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This scripts takes an existing, active yocto build, and puts pieces of it into a bundle directory, which is then placed in a tarball, and that goes into a self-extracting shell archive. This allows moving the bundle to another machine, or placing it on a network server for downloads. It's like esdk bundles, except inside there is a regular, plain, direct yocto build.
The bundle includes:
- copies of all layers used in a build, with their git histories
- build configuration as a template in a special layer private to the bundle
- (this is the best bit) all the sstate needed for the specified target, but *only* that sstate and not a complete copy of the local cache.
When someone runs the self-extracting shell archive, everything should 'just work': the layers are restored, and the build
set up to use them, the bundled sstate cache, and the template configuration.
Alternatively, it's possible to not include the cache, and presumably it will come from a configured sstate mirror.
This enables at least two*scratch*three interesting use cases:
- moving builds around in single-step fashion: run the script, and give the artifact to someone else via file sharing or usb sticks.
- publishing builds on the network, perhaps right next to the sstate cache that can fulfil them. This allows an alternative, much smoother entry point into 'doing yocto', as this guarantees availability of needed cache items, and so builds will be a lot faster than the typical 'compile everything from scratch' yocto experience. Sstate is guaranteed to be available, as there's a record of needed items in each bundle, and that record can be used to prevent pruning of still-needed cache items.
- this, in turn, can enable 'binary Yocto distro' implemented with sstate, if there's a higher level tool that creates bundles in a systematic, structured fashion, and another tool that iterates over available bundles, and lets users pick them for building locally.
--- meta/files/bundle-shar-extract.sh | 55 +++++++++++++++ meta/lib/oeqa/selftest/cases/sstatetests.py | 13 ++++ scripts/oe-replicate-build | 75 +++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 meta/files/bundle-shar-extract.sh create mode 100755 scripts/oe-replicate-build
meta/files/bundle-shar-extract.sh new file +55 -0
@@ -0,0 +1,55 @@ +#!/bin/bash + +verbose=0 +listcontents=0 +prepare_buildsystem="yes" +while getopts "Dln" OPT; do + case $OPT in + D) + verbose=1 + ;; + l) + listcontents=1 + ;; + n) + prepare_buildsystem="no" + ;; + *) + echo "Usage: $(basename "$0") [-D] [-l] [-n]" + echo " -D Use set -x to see what is going on" + echo " -l list files that will be extracted" + echo " -n Extract files but do not prepare the build system" + exit 1 + ;; + esac +done + +if [ $verbose = 1 ] ; then + set -x +fi + +payload_offset=$(($(grep -na -m1 "^MARKER:$" "$0"|cut -d':' -f1) + 1)) + +if [ "$listcontents" = "1" ] ; then + tail -n +$payload_offset "$0"| tar tv || exit 1 + exit +fi + + +tail -n +$payload_offset "$0"| tar mx --zstd --checkpoint=.2500 || exit 1 + +if [ $prepare_buildsystem = "no" ] ; then + exit +fi + +target_dir=$(basename "$0" .sh) +pushd $target_dir +layers/setup-build setup -c build-config-default -b build --no-shell > setup-build.log +popd + +echo "Each time you wish to use this build in a new shell session, you need to source the environment setup script:" +echo " \$ . $target_dir/build/init-build-env" + +exit 0 + +MARKER:
meta/lib/oeqa/selftest/cases/sstatetests.py +13 -0
@@ -1007,3 +1007,16 @@ MACHINE = "{}" def test_local_cache_qemuarm64(self): exceptions = [] self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) + +class SStateBundles(SStateCheckObjectPresence): + def test_minimal_bundle(self): + targets = "core-image-minimal" + machine = get_bb_var('MACHINE') + bitbake("--runall build {}".format(targets)) + runCmd("oe-replicate-build --targets {}".format(targets)) + extractedbundledir = tempfile.mkdtemp(prefix='bundle-extracted-', dir=self.topdir) + runCmd("../build-bundle.sh", cwd=extractedbundledir) + result = runCmd(". build-bundle/build/init-build-env && MACHINE={} bitbake -DD -n {}".format(machine,targets), cwd=extractedbundledir, shell=True, executable='/bin/bash') + + exceptions = [] + self.check_bb_output(result.output, targets, exceptions, check_cdn=False)
scripts/oe-replicate-build new file +75 -0
75 lines not on this page · raw message
Signature · 1 line
2.39.2