Skip to contentSkip to conversation

license/license_image: Fix license file layout to avoid overlapping files

PATCH New Richard Purdie 1 participant 1 message Sep 19, 2023

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 license/license_image: Fix license file layout to avoid overlapping files

Currently DEPLOY_DIR/licenses is added to SSTATE_ALLOW_OVERLAP_FILES. This leads to bugs since when one MACHINE_ARCH recipes is cleaned, it removes the files for another which then results in later build failures as license files disappear.

The solution is to include SSTAGE_PKGARCH in the path names to the license files. That does mean a search has to be used to find the correct license files for a given PN but that can be done via SSTATE_ARCHS.

The implication for other tools is the layout has changed so tools will need to adapt to the new paths. The benefit is no more strange build failures such as from patterns like:

MACHINE=qemux86-64 bitbake core-image-minimal MACHINE=genericx86-64 bitbake core-image-minimal MACHINE=qemux86-64 bitbake linux-yocto -c clean MACHINE=genericx86-64 bitbake core-image-minimal -C rootfs

[YOCTO #14123]

For anyone finding this commit, I'd question how much people should be relying on this code for tooling and suggest the SPDX manifests should be the preferred data format going forward anyway.

---
 meta/classes-global/license.bbclass       |  2 +-
 meta/classes-global/sstate.bbclass        |  2 --
 meta/classes-recipe/license_image.bbclass | 35 ++++++++++++++++-------
 3 files changed, 26 insertions(+), 13 deletions(-)
meta/classes-global/license.bbclass +1 -1
index 23625f0104f..b2e0d3fabaf 100644
@@ -29,7 +29,7 @@ python do_populate_lic() {
     lic_files_paths = find_license_files(d)
 
     # The base directory we wrangle licenses to
-    destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN'))
+    destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('SSTATE_PKGARCH'), d.getVar('PN'))
     copy_license_files(lic_files_paths, destdir)
     info = get_recipe_info(d)
     with open(os.path.join(destdir, "recipeinfo"), "w") as f:
meta/classes-global/sstate.bbclass +0 -2
index c63e6730ea1..69163f5b7ca 100644
@@ -55,8 +55,6 @@ PV[vardepvalue] = "${PV}"
 SSTATE_EXTRAPATH[vardepvalue] = ""
 SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
 
-# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
-SSTATE_ALLOW_OVERLAP_FILES = "${DEPLOY_DIR}/licenses/"
 # Avoid docbook/sgml catalog warnings for now
 SSTATE_ALLOW_OVERLAP_FILES += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
 # sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
meta/classes-recipe/license_image.bbclass +25 -10
index fc859c7c659..838e13803e4 100644
@@ -18,7 +18,7 @@ python() {
 
 python write_package_manifest() {
     # Get list of installed packages
-    license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}')
+    license_image_dir = d.expand('${LICENSE_DIRECTORY}/${SSTATE_PKGARCH}/${IMAGE_NAME}')
     bb.utils.mkdirhier(license_image_dir)
     from oe.rootfs import image_list_installed_packages
     from oe.utils import format_pkg_list
@@ -49,7 +49,7 @@ python license_create_manifest() {
             pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
 
     rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
-                        d.getVar('IMAGE_NAME'), 'license.manifest')
+                        d.getVar('SSTATE_PKGARCH'), d.getVar('IMAGE_NAME'), 'license.manifest')
     write_license_files(d, rootfs_license_manifest, pkg_dic, rootfs=True)
 }
 
@@ -98,9 +98,13 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
                 license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
 
             for lic in pkg_dic[pkg]["LICENSES"]:
-                lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
-                                        pkg_dic[pkg]["PN"], "generic_%s" % 
-                                        re.sub(r'\+', '', lic))
+                for pkgarch in d.getVar("SSTATE_ARCHS").split():
+                    lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+                                            pkgarch,
+                                            pkg_dic[pkg]["PN"], "generic_%s" %
+                                            re.sub(r'\+', '', lic))
+                    if os.path.exists(lic_file):
+                        break
                 # add explicity avoid of CLOSED license because isn't generic
                 if lic == "CLOSED":
                    continue
@@ -130,8 +134,13 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
             for pkg in sorted(pkg_dic):
                 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
                 bb.utils.mkdirhier(pkg_rootfs_license_dir)
-                pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
-                                            pkg_dic[pkg]["PN"]) 
+                for pkgarch in d.getVar("SSTATE_ARCHS").split():
+                    pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+                                                   pkgarch, pkg_dic[pkg]["PN"])
+                    if os.path.exists(pkg_license_dir):
+                        break
+                if not os.path.exists(pkg_license_dir ):
+                    bb.fatal("Couldn't find license information for dependency %s" % pkg)
 
                 pkg_manifest_licenses = [canonical_license(d, lic) \
                         for lic in pkg_dic[pkg]["LICENSES"]]
@@ -204,12 +213,18 @@ def license_deployed_manifest(d):
         man_dic[dep]["PN"] = dep
         man_dic[dep]["FILES"] = \
             " ".join(get_deployed_files(dep_dic[dep]))
-        with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
+        for pkgarch in d.getVar("SSTATE_ARCHS").split():
+            licfile = os.path.join(lic_dir, pkgarch, dep, "recipeinfo")
+            if os.path.exists(licfile):
+                break
+        if not os.path.exists(licfile):
+            bb.fatal("Couldn't find license information for dependency %s" % dep)
+        with open(licfile, "r") as f:
             for line in f.readlines():
                 key,val = line.split(": ", 1)
                 man_dic[dep][key] = val[:-1]
 
-    lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+    lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'), d.getVar('SSTATE_PKGARCH'),
                                     d.getVar('IMAGE_NAME'))
     bb.utils.mkdirhier(lic_manifest_dir)
     image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
@@ -217,7 +232,7 @@ def license_deployed_manifest(d):
 
     link_name = d.getVar('IMAGE_LINK_NAME')
     if link_name:
-        lic_manifest_symlink_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+        lic_manifest_symlink_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'), d.getVar('SSTATE_PKGARCH'),
                                     link_name)
         # remove old symlink
         if os.path.islink(lic_manifest_symlink_dir):
Signature · 1 line
2.39.2