archiver.bbclass: Properly remove artifacts when configuration changes
PATCH 1/3 archiver.bbclass: Properly remove artifacts when configuration changes
Before, the different archiver tasks that produce artifacts would just add to the common output directory. This meant that changing how the archiver is configured would just add more artifacts, but never remove any. E.g., if "dumpdata" was enabled, it would add the environment data for each built recipe to the artifacts. However, if it was then disabled, all the generated artifacts would remain until each recipe was manually cleaned, or the entire tmp directory was removed.
This adds a task that cleans the output directory if needed. It is a separate task that all the other archiver tasks depend on because they all write to the same output directory so it must only be cleaned once. This also means it has to depend on all variables that affect any of the other tasks.
--- meta/classes/archiver.bbclass | 27 +++++++++++++---- meta/lib/oeqa/selftest/cases/archiver.py | 38 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-)
meta/classes/archiver.bbclass +21 -6
@@ -179,6 +179,20 @@ python () { d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn) } +do_ar_prepare[vardeps] += " \ + ARCHIVER_MODE \ + ARCHIVER_MIRROR_EXCLUDE \ + COPYLEFT_LICENSE_EXCLUDE \ + COPYLEFT_LICENSE_INCLUDE \ + COPYLEFT_PN_EXCLUDE \ + COPYLEFT_PN_INCLUDE \ + COPYLEFT_RECIPE_TYPES \ +" +do_ar_prepare[cleandirs] = "${ARCHIVER_TOPDIR}" +do_ar_prepare() { + : +} + # Take all the sources for a recipe and put them in WORKDIR/archiver-work/. # Files in SRC_URI are copied directly, anything that's a directory # (e.g. git repositories) is "unpacked" and then put into a tarball.
@@ -609,14 +623,15 @@ do_deploy_archives[sstate-inputdirs] = "${ARCHIVER_TOPDIR}" do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}" addtask do_deploy_archives_setscene -addtask do_ar_original after do_unpack -addtask do_unpack_and_patch after do_patch do_preconfigure +addtask do_ar_prepare +addtask do_ar_original after do_unpack do_ar_prepare +addtask do_unpack_and_patch after do_patch do_preconfigure do_ar_prepare addtask do_ar_patched after do_unpack_and_patch addtask do_ar_configured after do_unpack_and_patch -addtask do_ar_mirror after do_fetch -addtask do_dumpdata -addtask do_ar_recipe -addtask do_deploy_archives +addtask do_ar_mirror after do_fetch do_ar_prepare +addtask do_dumpdata after do_ar_prepare +addtask do_ar_recipe after do_ar_prepare +addtask do_deploy_archives after do_ar_prepare do_build[recrdeptask] += "do_deploy_archives" do_rootfs[recrdeptask] += "do_deploy_archives" do_populate_sdk[recrdeptask] += "do_deploy_archives"
meta/lib/oeqa/selftest/cases/archiver.py +38 -0
@@ -343,3 +343,41 @@ class Archiver(OESelftestTestCase): ]: target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name) self.assertTrue(os.path.exists(target_path)) + + def test_archiver_cleanup(self): + """ + Test that the archiver removes no longer needed artifacts when its + configuration is modified. + """ + + target = 'selftest-ed-native' + target_file_name = 'selftest-ed-native-1.21.1-r0-showdata.dump' + + def assert_dumpdata_present(expect_present): + bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'BUILD_SYS']) + glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['BUILD_SYS'], '%s-*' % target) + glob_result = glob.glob(glob_str) + self.assertTrue(glob_result, 'Missing archiver directory for %s' % target) + + archive_path = os.path.join(glob_result[0], target_file_name) + if expect_present: + self.assertTrue(os.path.exists(archive_path), + 'Missing archive file %s' % target_file_name) + else: + self.assertFalse(os.path.exists(archive_path), + 'Unexpected archive file %s' % target_file_name) + + features = 'INHERIT += "archiver"\n' + self.write_config(features) + bitbake('-c deploy_archives %s -f' % target) + assert_dumpdata_present(False) + + features += 'ARCHIVER_MODE[dumpdata] = "1"\n' + self.write_config(features) + bitbake('-c deploy_archives %s -f' % target) + assert_dumpdata_present(True) + + features = 'INHERIT += "archiver"\n' + self.write_config(features) + bitbake('-c deploy_archives %s -f' % target) + assert_dumpdata_present(False)
PATCH 2/3 oeqa/selftest/archiver: Do not run bitbake -c clean unnecessarily
Now that the archiver bbclass properly adapts its output based on its configuration, it should no longer be needed to run bitbake -c clean before invoking the archiver. There is one exception, which is due to modifying DL_DIR as it does not automatically trigger a re-fetch.
--- meta/lib/oeqa/selftest/cases/archiver.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
meta/lib/oeqa/selftest/cases/archiver.py +2 -7
@@ -31,7 +31,6 @@ class Archiver(OESelftestTestCase): features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe self.write_config(features) - bitbake('-c clean %s %s' % (include_recipe, exclude_recipe)) bitbake("-c deploy_archives %s %s" % (include_recipe, exclude_recipe)) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
@@ -62,7 +61,6 @@ class Archiver(OESelftestTestCase): features += 'COPYLEFT_RECIPE_TYPES = "target"\n' self.write_config(features) - bitbake('-c clean %s %s' % (target_recipe, native_recipe)) bitbake("%s -c deploy_archives %s" % (target_recipe, native_recipe)) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
@@ -99,7 +97,6 @@ class Archiver(OESelftestTestCase): features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % target_recipes[1] self.write_config(features) - bitbake('-c clean %s %s' % (' '.join(target_recipes), ' '.join(native_recipes))) bitbake('-c deploy_archives %s %s' % (' '.join(target_recipes), ' '.join(native_recipes))) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
@@ -174,7 +171,6 @@ class Archiver(OESelftestTestCase): features += extra_config self.write_config(features) - bitbake('-c clean %s' % (target)) bitbake('-c deploy_archives %s' % (target)) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'BUILD_SYS'])
@@ -255,7 +251,6 @@ class Archiver(OESelftestTestCase): features += 'ARCHIVER_MIRROR_EXCLUDE = "${GNU_MIRROR}"\n' self.write_config(features) - bitbake('-c clean %s' % (target)) bitbake('-c deploy_archives %s' % (target)) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
@@ -281,7 +276,6 @@ class Archiver(OESelftestTestCase): self.write_config(features) for target in ['selftest-ed', 'selftest-hardlink']: - bitbake('-c clean %s' % (target)) bitbake('-c deploy_archives %s' % (target)) bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
@@ -302,7 +296,6 @@ class Archiver(OESelftestTestCase): features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n' self.write_config(features) - bitbake('-c clean git-submodule-test') bitbake('-c deploy_archives -f git-submodule-test') bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
@@ -330,6 +323,8 @@ class Archiver(OESelftestTestCase): features += 'DL_DIR = "${TOPDIR}/downloads-shallow"\n' self.write_config(features) + # Clean the build directory so that fetch is rerun. + # This is needed due to modifying DL_DIR. bitbake('-c clean git-submodule-test') bitbake('-c deploy_archives -f git-submodule-test')
PATCH 3/3 archiver.bbclass: Add ARCHIVER_MIRROR_INCLUDE
In reply to Peter KjellerstedtWhen using the archiver's mirror mode, ARCHIVER_MIRROR_INCLUDE contains a list of URI prefixes to always include in the mirror. This may be useful if recipes that are included in the mirror (due to their licenses) depend on some recipe that otherwise would not be included in the mirror, and that recipe uses files that are normally only available on a private server.
--- meta/classes/archiver.bbclass | 37 ++++++++++++++++++++++-- meta/lib/oeqa/selftest/cases/archiver.py | 24 +++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-)
meta/classes/archiver.bbclass +34 -3
@@ -35,7 +35,13 @@ # ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single # directory suitable for direct use as a mirror. Duplicate sources are # ignored. -# 12) Source mirror exclusions: +# 12) Source mirror inclusions: +# ARCHIVER_MIRROR_INCLUDE is a list of URI prefixes to always include in +# the mirror. This may be useful if recipes that are included in the mirror +# (due to their licenses) depend on some recipe that otherwise would not be +# included in the mirror, and that recipe uses files that are normally only +# available on a private server. +# 13) Source mirror exclusions: # ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror. # This may be used for sources which you are already publishing yourself # (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
@@ -77,8 +83,9 @@ do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}" # This is a convenience for the shell script to use it def include_package(d, pn): - - included, reason = copyleft_should_include(d) + included, reason = archiver_should_include(d) + if not included: + included, reason = copyleft_should_include(d) if not included: bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason)) return False
@@ -99,6 +106,29 @@ def include_package(d, pn): return True +def archiver_should_include(d): + ar_src = d.getVarFlag('ARCHIVER_MODE', 'src') + if ar_src == "mirror": + src_uri = (d.getVar('SRC_URI') or '').split() + if len(src_uri) == 0: + return False, None + + mirror_inclusions = (d.getVar('ARCHIVER_MIRROR_INCLUDE') or '').split() + + def is_included(url): + for prefix in mirror_inclusions: + if url.startswith(prefix): + return True + return False + + fetcher = bb.fetch2.Fetch(src_uri, d) + + for ud in fetcher.expanded_urldata(): + if is_included(ud.url): + return True, "URL matches ARCHIVER_MIRROR_INCLUDE" + + return False, None + python () { pn = d.getVar('PN') assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
@@ -182,6 +212,7 @@ python () { do_ar_prepare[vardeps] += " \ ARCHIVER_MODE \ ARCHIVER_MIRROR_EXCLUDE \ + ARCHIVER_MIRROR_INCLUDE \ COPYLEFT_LICENSE_EXCLUDE \ COPYLEFT_LICENSE_INCLUDE \ COPYLEFT_PN_EXCLUDE \
meta/lib/oeqa/selftest/cases/archiver.py +24 -0
@@ -261,6 +261,30 @@ class Archiver(OESelftestTestCase): archive_path = os.path.join(glob_result[0], target_file_name) self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name)) + def test_archiver_mode_mirror_include(self): + """ + Test that `ARCHIVER_MIRROR_INCLUDE` causes a source URL to be included + in the mirror even when the recipe would otherwise be excluded by the + copyleft license filter. + """ + + target = 'selftest-ed' + target_file_name = 'ed-1.21.1.tar.lz' + + features = 'INHERIT += "archiver"\n' + features += 'ARCHIVER_MODE[src] = "mirror"\n' + features += 'ARCHIVER_MODE[mirror] = "combined"\n' + features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n' + features += 'COPYLEFT_LICENSE_INCLUDE = "CLOSED"\n' + features += 'ARCHIVER_MIRROR_INCLUDE = "${GNU_MIRROR}"\n' + self.write_config(features) + + bitbake('-c deploy_archives %s' % (target)) + + bb_vars = get_bb_vars(['DEPLOY_DIR_SRC']) + target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name) + self.assertTrue(os.path.exists(target_path), 'Missing archive file %s' % (target_file_name)) + def test_archiver_mode_mirror_combined(self): """ Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
Re: [OE-core] [PATCH 3/3] archiver.bbclass: Add ARCHIVER_MIRROR_INCLUDE
Quoted 7 lines · Peter Kjellerstedt via lists.openembedded.org
When using the archiver's mirror mode,
ARCHIVER_MIRROR_INCLUDEcontains a list of URI prefixes to always include in the mirror. This may be useful if recipes that are included in the mirror (due to their licenses) depend on some recipe that otherwise would not be included in the mirror, and that recipe uses files that are normally only available on a private server.
This looks useful, but I don’t see any discussion of who wins if INCLUDE and EXCLUDE both match some prefixes.
Also I’d have expected the implementation for the include and exclude logic to be done at roughly the same point in the code but this doesn’t seem to be the case. Does the code need some general refactoring, or is there some complex logic required due to how it has to work?
Cheers,
Ross
Quoted 119 lines
Signed-off-by Peter Kjellerstedt <[email protected]>---
meta/classes/archiver.bbclass | 37 ++++++++++++++++++++++--meta/lib/oeqa/selftest/cases/archiver.py| 24 +++++++++++++++2 files changed, 58 insertions(+), 3 deletions(-)meta/classes/archiver.bbclass +7
-1@@ -35,7 +35,13 @@ # ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single # directory suitable for direct use as a mirror. Duplicate sources are # ignored. -# 12) Source mirror exclusions: +# 12) Source mirror inclusions: +# ARCHIVER_MIRROR_INCLUDE is a list of URI prefixes to always include in +# the mirror. This may be useful if recipes that are included in the mirror +# (due to their licenses) depend on some recipe that otherwise would not be +# included in the mirror, and that recipe uses files that are normally only +# available on a private server. +# 13) Source mirror exclusions: # ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror. # This may be used for sources which you are already publishing yourself # (e.g. if the URI starts with 'https://mysite.com/' and your mirror is@@ -77,8 +83,9 @@ do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}" # This is a convenience for the shell script to use itdef
include_package(d, pn):meta/classes/archiver.bbclass +27
-2- - included, reason = copyleft_should_include(d) + included, reason = archiver_should_include(d) + if not included: + included, reason = copyleft_should_include(d) if not included: bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason)) return False@@ -99,6 +106,29 @@ def include_package(d, pn): return True +def archiver_should_include(d): + ar_src = d.getVarFlag('ARCHIVER_MODE', 'src') + if ar_src == "mirror": + src_uri = (d.getVar('SRC_URI') or '').split() + if len(src_uri) == 0: + return False, None + + mirror_inclusions = (d.getVar('ARCHIVER_MIRROR_INCLUDE') or '').split() + + def is_included(url): + for prefix in mirror_inclusions: + if url.startswith(prefix): + return True + return False + + fetcher = bb.fetch2.Fetch(src_uri, d) + + for ud in fetcher.expanded_urldata(): + if is_included(ud.url): + return True, "URL matches ARCHIVER_MIRROR_INCLUDE" + + return False, None + python () { pn = d.getVar('PN') assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()@@ -182,6 +212,7 @@ python () { do_ar_prepare[vardeps] += " \ ARCHIVER_MODE \ ARCHIVER_MIRROR_EXCLUDE \ + ARCHIVER_MIRROR_INCLUDE \ COPYLEFT_LICENSE_EXCLUDE \ COPYLEFT_LICENSE_INCLUDE \ COPYLEFT_PN_EXCLUDE \meta/lib/oeqa/selftest/cases/archiver.py +24
-1@@ -261,6 +261,30 @@ class Archiver(OESelftestTestCase): archive_path = os.path.join(glob_result[0], target_file_name) self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name)) + def test_archiver_mode_mirror_include(self): + """ + Test that `ARCHIVER_MIRROR_INCLUDE` causes a source URL to be included + in the mirror even when the recipe would otherwise be excluded by the + copyleft license filter. + """ + + target = 'selftest-ed' + target_file_name = 'ed-1.21.1.tar.lz' + + features = 'INHERIT += "archiver"\n' + features += 'ARCHIVER_MODE[src] = "mirror"\n' + features += 'ARCHIVER_MODE[mirror] = "combined"\n' + features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n' + features += 'COPYLEFT_LICENSE_INCLUDE = "CLOSED"\n' + features += 'ARCHIVER_MIRROR_INCLUDE = "${GNU_MIRROR}"\n' + self.write_config(features) + + bitbake('-c deploy_archives %s' % (target)) + + bb_vars = get_bb_vars(['DEPLOY_DIR_SRC']) + target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name) + self.assertTrue(os.path.exists(target_path), 'Missing archive file %s' % (target_file_name)) + def test_archiver_mode_mirror_combined(self): """ Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"` -=-=-=-=-=-=-=-=-=-=-=-Links: You receive all messages sent to this group.
View/Reply Online (#238255): https://lists.openembedded.org/g/openembedded-core/message/238255
Mute This Topic: https://lists.openembedded.org/mt/119710236/6875888
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-