Skip to contentSkip to conversation

archiver.bbclass: Properly remove artifacts when configuration changes

PATCH In review Peter Kjellerstedt 2 participants 4 messages Jun 8 – Jun 19

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 archiver.bbclass: Properly remove artifacts when configuration changes

Before, the different archiver tasks that produce artifacts would just add to the common output directory. This meant that changing how the archiver is configured would just add more artifacts, but never remove any. E.g., if "dumpdata" was enabled, it would add the environment data for each built recipe to the artifacts. However, if it was then disabled, all the generated artifacts would remain until each recipe was manually cleaned, or the entire tmp directory was removed.

This adds a task that cleans the output directory if needed. It is a separate task that all the other archiver tasks depend on because they all write to the same output directory so it must only be cleaned once. This also means it has to depend on all variables that affect any of the other tasks.

---
 meta/classes/archiver.bbclass            | 27 +++++++++++++----
 meta/lib/oeqa/selftest/cases/archiver.py | 38 ++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 6 deletions(-)
meta/classes/archiver.bbclass +21 -6
index 1f1ee45bd7..a3f8689b19 100644
@@ -179,6 +179,20 @@ python () {
                 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn)
 }
 
+do_ar_prepare[vardeps] += " \
+    ARCHIVER_MODE \
+    ARCHIVER_MIRROR_EXCLUDE \
+    COPYLEFT_LICENSE_EXCLUDE \
+    COPYLEFT_LICENSE_INCLUDE \
+    COPYLEFT_PN_EXCLUDE \
+    COPYLEFT_PN_INCLUDE \
+    COPYLEFT_RECIPE_TYPES \
+"
+do_ar_prepare[cleandirs] = "${ARCHIVER_TOPDIR}"
+do_ar_prepare() {
+	:
+}
+
 # Take all the sources for a recipe and put them in WORKDIR/archiver-work/.
 # Files in SRC_URI are copied directly, anything that's a directory
 # (e.g. git repositories) is "unpacked" and then put into a tarball.
@@ -609,14 +623,15 @@ do_deploy_archives[sstate-inputdirs] = "${ARCHIVER_TOPDIR}"
 do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}"
 addtask do_deploy_archives_setscene
 
-addtask do_ar_original after do_unpack
-addtask do_unpack_and_patch after do_patch do_preconfigure
+addtask do_ar_prepare
+addtask do_ar_original after do_unpack do_ar_prepare
+addtask do_unpack_and_patch after do_patch do_preconfigure do_ar_prepare
 addtask do_ar_patched after do_unpack_and_patch
 addtask do_ar_configured after do_unpack_and_patch
-addtask do_ar_mirror after do_fetch
-addtask do_dumpdata
-addtask do_ar_recipe
-addtask do_deploy_archives
+addtask do_ar_mirror after do_fetch do_ar_prepare
+addtask do_dumpdata after do_ar_prepare
+addtask do_ar_recipe after do_ar_prepare
+addtask do_deploy_archives after do_ar_prepare
 do_build[recrdeptask] += "do_deploy_archives"
 do_rootfs[recrdeptask] += "do_deploy_archives"
 do_populate_sdk[recrdeptask] += "do_deploy_archives"
meta/lib/oeqa/selftest/cases/archiver.py +38 -0
index 82b0293338..872504808a 100644
@@ -343,3 +343,41 @@ class Archiver(OESelftestTestCase):
         ]:
             target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
             self.assertTrue(os.path.exists(target_path))
+
+    def test_archiver_cleanup(self):
+        """
+        Test that the archiver removes no longer needed artifacts when its
+        configuration is modified.
+        """
+
+        target = 'selftest-ed-native'
+        target_file_name = 'selftest-ed-native-1.21.1-r0-showdata.dump'
+
+        def assert_dumpdata_present(expect_present):
+            bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'BUILD_SYS'])
+            glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['BUILD_SYS'], '%s-*' % target)
+            glob_result = glob.glob(glob_str)
+            self.assertTrue(glob_result, 'Missing archiver directory for %s' % target)
+
+            archive_path = os.path.join(glob_result[0], target_file_name)
+            if expect_present:
+                self.assertTrue(os.path.exists(archive_path),
+                                'Missing archive file %s' % target_file_name)
+            else:
+                self.assertFalse(os.path.exists(archive_path),
+                                 'Unexpected archive file %s' % target_file_name)
+
+        features = 'INHERIT += "archiver"\n'
+        self.write_config(features)
+        bitbake('-c deploy_archives %s -f' % target)
+        assert_dumpdata_present(False)
+
+        features += 'ARCHIVER_MODE[dumpdata] = "1"\n'
+        self.write_config(features)
+        bitbake('-c deploy_archives %s -f' % target)
+        assert_dumpdata_present(True)
+
+        features = 'INHERIT += "archiver"\n'
+        self.write_config(features)
+        bitbake('-c deploy_archives %s -f' % target)
+        assert_dumpdata_present(False)
#1 lore raw

PATCH 2/3 oeqa/selftest/archiver: Do not run bitbake -c clean unnecessarily

Now that the archiver bbclass properly adapts its output based on its configuration, it should no longer be needed to run bitbake -c clean before invoking the archiver. There is one exception, which is due to modifying DL_DIR as it does not automatically trigger a re-fetch.

---
 meta/lib/oeqa/selftest/cases/archiver.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)
meta/lib/oeqa/selftest/cases/archiver.py +2 -7
index 872504808a..122d16b655 100644
@@ -31,7 +31,6 @@ class Archiver(OESelftestTestCase):
         features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe
         self.write_config(features)
 
-        bitbake('-c clean %s %s' % (include_recipe, exclude_recipe))
         bitbake("-c deploy_archives %s %s" % (include_recipe, exclude_recipe))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
@@ -62,7 +61,6 @@ class Archiver(OESelftestTestCase):
         features += 'COPYLEFT_RECIPE_TYPES = "target"\n'
         self.write_config(features)
 
-        bitbake('-c clean %s %s' % (target_recipe, native_recipe))
         bitbake("%s -c deploy_archives %s" % (target_recipe, native_recipe))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
@@ -99,7 +97,6 @@ class Archiver(OESelftestTestCase):
         features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % target_recipes[1]
         self.write_config(features)
 
-        bitbake('-c clean %s %s' % (' '.join(target_recipes), ' '.join(native_recipes)))
         bitbake('-c deploy_archives %s %s' % (' '.join(target_recipes), ' '.join(native_recipes)))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
@@ -174,7 +171,6 @@ class Archiver(OESelftestTestCase):
             features += extra_config
         self.write_config(features)
 
-        bitbake('-c clean %s' % (target))
         bitbake('-c deploy_archives %s' % (target))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'BUILD_SYS'])
@@ -255,7 +251,6 @@ class Archiver(OESelftestTestCase):
         features += 'ARCHIVER_MIRROR_EXCLUDE = "${GNU_MIRROR}"\n'
         self.write_config(features)
 
-        bitbake('-c clean %s' % (target))
         bitbake('-c deploy_archives %s' % (target))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
@@ -281,7 +276,6 @@ class Archiver(OESelftestTestCase):
         self.write_config(features)
 
         for target in ['selftest-ed', 'selftest-hardlink']:
-            bitbake('-c clean %s' % (target))
             bitbake('-c deploy_archives %s' % (target))
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
@@ -302,7 +296,6 @@ class Archiver(OESelftestTestCase):
         features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
         self.write_config(features)
 
-        bitbake('-c clean git-submodule-test')
         bitbake('-c deploy_archives -f git-submodule-test')
 
         bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
@@ -330,6 +323,8 @@ class Archiver(OESelftestTestCase):
         features += 'DL_DIR = "${TOPDIR}/downloads-shallow"\n'
         self.write_config(features)
 
+        # Clean the build directory so that fetch is rerun.
+        # This is needed due to modifying DL_DIR.
         bitbake('-c clean git-submodule-test')
         bitbake('-c deploy_archives -f git-submodule-test')
 
#2 lore raw

PATCH 3/3 archiver.bbclass: Add ARCHIVER_MIRROR_INCLUDE

In reply to Peter Kjellerstedt

When using the archiver's mirror mode, ARCHIVER_MIRROR_INCLUDE contains a list of URI prefixes to always include in the mirror. This may be useful if recipes that are included in the mirror (due to their licenses) depend on some recipe that otherwise would not be included in the mirror, and that recipe uses files that are normally only available on a private server.

---
 meta/classes/archiver.bbclass            | 37 ++++++++++++++++++++++--
 meta/lib/oeqa/selftest/cases/archiver.py | 24 +++++++++++++++
 2 files changed, 58 insertions(+), 3 deletions(-)
meta/classes/archiver.bbclass +34 -3
index a3f8689b19..035d0dce0f 100644
@@ -35,7 +35,13 @@
 #     ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single
 #     directory suitable for direct use as a mirror. Duplicate sources are
 #     ignored.
-# 12) Source mirror exclusions:
+# 12) Source mirror inclusions:
+#     ARCHIVER_MIRROR_INCLUDE is a list of URI prefixes to always include in
+#     the mirror. This may be useful if recipes that are included in the mirror
+#     (due to their licenses) depend on some recipe that otherwise would not be
+#     included in the mirror, and that recipe uses files that are normally only
+#     available on a private server.
+# 13) Source mirror exclusions:
 #     ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror.
 #     This may be used for sources which you are already publishing yourself
 #     (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
@@ -77,8 +83,9 @@ do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
 # This is a convenience for the shell script to use it
 
 def include_package(d, pn):
-
-    included, reason = copyleft_should_include(d)
+    included, reason = archiver_should_include(d)
+    if not included:
+        included, reason = copyleft_should_include(d)
     if not included:
         bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason))
         return False
@@ -99,6 +106,29 @@ def include_package(d, pn):
 
     return True
 
+def archiver_should_include(d):
+    ar_src = d.getVarFlag('ARCHIVER_MODE', 'src')
+    if ar_src == "mirror":
+        src_uri = (d.getVar('SRC_URI') or '').split()
+        if len(src_uri) == 0:
+            return False, None
+
+        mirror_inclusions = (d.getVar('ARCHIVER_MIRROR_INCLUDE') or '').split()
+
+        def is_included(url):
+            for prefix in mirror_inclusions:
+                if url.startswith(prefix):
+                    return True
+            return False
+
+        fetcher = bb.fetch2.Fetch(src_uri, d)
+
+        for ud in fetcher.expanded_urldata():
+            if is_included(ud.url):
+                return True, "URL matches ARCHIVER_MIRROR_INCLUDE"
+
+    return False, None
+
 python () {
     pn = d.getVar('PN')
     assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
@@ -182,6 +212,7 @@ python () {
 do_ar_prepare[vardeps] += " \
     ARCHIVER_MODE \
     ARCHIVER_MIRROR_EXCLUDE \
+    ARCHIVER_MIRROR_INCLUDE \
     COPYLEFT_LICENSE_EXCLUDE \
     COPYLEFT_LICENSE_INCLUDE \
     COPYLEFT_PN_EXCLUDE \
meta/lib/oeqa/selftest/cases/archiver.py +24 -0
index 122d16b655..e9f178d5c9 100644
@@ -261,6 +261,30 @@ class Archiver(OESelftestTestCase):
         archive_path = os.path.join(glob_result[0], target_file_name)
         self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name))
 
+    def test_archiver_mode_mirror_include(self):
+        """
+        Test that `ARCHIVER_MIRROR_INCLUDE` causes a source URL to be included
+        in the mirror even when the recipe would otherwise be excluded by the
+        copyleft license filter.
+        """
+
+        target = 'selftest-ed'
+        target_file_name = 'ed-1.21.1.tar.lz'
+
+        features = 'INHERIT += "archiver"\n'
+        features += 'ARCHIVER_MODE[src] = "mirror"\n'
+        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
+        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
+        features += 'COPYLEFT_LICENSE_INCLUDE = "CLOSED"\n'
+        features += 'ARCHIVER_MIRROR_INCLUDE = "${GNU_MIRROR}"\n'
+        self.write_config(features)
+
+        bitbake('-c deploy_archives %s' % (target))
+
+        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
+        target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
+        self.assertTrue(os.path.exists(target_path), 'Missing archive file %s' % (target_file_name))
+
     def test_archiver_mode_mirror_combined(self):
         """
         Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
#3 lore raw

Re: [OE-core] [PATCH 3/3] archiver.bbclass: Add ARCHIVER_MIRROR_INCLUDE

Quoted 7 lines · Peter Kjellerstedt via lists.openembedded.org

When using the archiver's mirror mode, ARCHIVER_MIRROR_INCLUDE contains a list of URI prefixes to always include in the mirror. This may be useful if recipes that are included in the mirror (due to their licenses) depend on some recipe that otherwise would not be included in the mirror, and that recipe uses files that are normally only available on a private server.

This looks useful, but I don’t see any discussion of who wins if INCLUDE and EXCLUDE both match some prefixes.

Also I’d have expected the implementation for the include and exclude logic to be done at roughly the same point in the code but this doesn’t seem to be the case. Does the code need some general refactoring, or is there some complex logic required due to how it has to work?

Cheers,
Ross

Quoted 119 lines

---
meta/classes/archiver.bbclass | 37 ++++++++++++++++++++++--
meta/lib/oeqa/selftest/cases/archiver.py | 24 +++++++++++++++

2 files changed, 58 insertions(+), 3 deletions(-)
meta/classes/archiver.bbclass +7 -1
index a3f8689b19..035d0dce0f 100644
@@ -35,7 +35,13 @@
 #     ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single
 #     directory suitable for direct use as a mirror. Duplicate sources are
 #     ignored.
-# 12) Source mirror exclusions:
+# 12) Source mirror inclusions:
+#     ARCHIVER_MIRROR_INCLUDE is a list of URI prefixes to always include in
+#     the mirror. This may be useful if recipes that are included in the mirror
+#     (due to their licenses) depend on some recipe that otherwise would not be
+#     included in the mirror, and that recipe uses files that are normally only
+#     available on a private server.
+# 13) Source mirror exclusions:
 #     ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror.
 #     This may be used for sources which you are already publishing yourself
 #     (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
@@ -77,8 +83,9 @@ do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
 # This is a convenience for the shell script to use it
 

def include_package(d, pn):

meta/classes/archiver.bbclass +27 -2
-
-    included, reason = copyleft_should_include(d)
+    included, reason = archiver_should_include(d)
+    if not included:
+        included, reason = copyleft_should_include(d)
    if not included:
        bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason))
        return False
@@ -99,6 +106,29 @@ def include_package(d, pn):
 
    return True
 
+def archiver_should_include(d):
+    ar_src = d.getVarFlag('ARCHIVER_MODE', 'src')
+    if ar_src == "mirror":
+        src_uri = (d.getVar('SRC_URI') or '').split()
+        if len(src_uri) == 0:
+            return False, None
+
+        mirror_inclusions = (d.getVar('ARCHIVER_MIRROR_INCLUDE') or '').split()
+
+        def is_included(url):
+            for prefix in mirror_inclusions:
+                if url.startswith(prefix):
+                    return True
+            return False
+
+        fetcher = bb.fetch2.Fetch(src_uri, d)
+
+        for ud in fetcher.expanded_urldata():
+            if is_included(ud.url):
+                return True, "URL matches ARCHIVER_MIRROR_INCLUDE"
+
+    return False, None
+
 python () {
    pn = d.getVar('PN')
    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
@@ -182,6 +212,7 @@ python () {
 do_ar_prepare[vardeps] += " \
    ARCHIVER_MODE \
    ARCHIVER_MIRROR_EXCLUDE \
+    ARCHIVER_MIRROR_INCLUDE \
    COPYLEFT_LICENSE_EXCLUDE \
    COPYLEFT_LICENSE_INCLUDE \
    COPYLEFT_PN_EXCLUDE \
meta/lib/oeqa/selftest/cases/archiver.py +24 -1
index 122d16b655..e9f178d5c9 100644
@@ -261,6 +261,30 @@ class Archiver(OESelftestTestCase):
        archive_path = os.path.join(glob_result[0], target_file_name)
        self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name))
 
+    def test_archiver_mode_mirror_include(self):
+        """
+        Test that `ARCHIVER_MIRROR_INCLUDE` causes a source URL to be included
+        in the mirror even when the recipe would otherwise be excluded by the
+        copyleft license filter.
+        """
+
+        target = 'selftest-ed'
+        target_file_name = 'ed-1.21.1.tar.lz'
+
+        features = 'INHERIT += "archiver"\n'
+        features += 'ARCHIVER_MODE[src] = "mirror"\n'
+        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
+        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
+        features += 'COPYLEFT_LICENSE_INCLUDE = "CLOSED"\n'
+        features += 'ARCHIVER_MIRROR_INCLUDE = "${GNU_MIRROR}"\n'
+        self.write_config(features)
+
+        bitbake('-c deploy_archives %s' % (target))
+
+        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
+        target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
+        self.assertTrue(os.path.exists(target_path), 'Missing archive file %s' % (target_file_name))
+
    def test_archiver_mode_mirror_combined(self):
        """
        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group.
View/Reply Online (#238255): https://lists.openembedded.org/g/openembedded-core/message/238255
Mute This Topic: https://lists.openembedded.org/mt/119710236/6875888
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-