Skip to contentSkip to conversation

INCOMPATIBLE_LICENSE re-work

PATCH New Saul Wold 1 participant 3 messages Feb 26, 2022

lore
Share & tools
lore

Saul Wold

author
#0 lore raw

PATCH v3 1/2 INCOMPATIBLE_LICENSE re-work

From: Saul Wold <[email protected]>

This re-writes the INCOMPATIBLE_LICENSE checking code to replace the WHITELIST_<lic> with INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:<lic> <pkg>:<lic> ...'

This initial set of changes leaves most of the code structure in place, but the code in base.bbclass needs to be re-written to make the check more consistent around packages (PKGS) and not recipe names (PN). This also is taking into account the changes for SPDX licenses.

---
 meta/classes/base.bbclass                     | 26 +++++++++---------
 meta/classes/license_image.bbclass            | 27 +++++++------------
 meta/classes/multilib.bbclass                 |  6 ++---
 meta/conf/bitbake.conf                        | 10 +++++++
 .../distro/include/default-distrovars.inc     |  2 +-
 .../oeqa/selftest/cases/incompatible_lic.py   | 10 +++----
 6 files changed, 43 insertions(+), 38 deletions(-)
meta/classes/base.bbclass +14 -12
index 55f654d37d0..ddca87d4a8c 100644
@@ -595,21 +595,23 @@ python () {
         if check_license and bad_licenses:
             bad_licenses = expand_wildcard_licenses(d, bad_licenses)
 
-            whitelist = []
-            for lic in bad_licenses:
-                spdx_license = return_spdx(d, lic)
-                whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-                if spdx_license:
-                    whitelist.extend((d.getVar("WHITELIST_" + spdx_license) or "").split())
-
-            if pn in whitelist:
+            exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
+
+            pkg_exceptions = {}
+            for exception in exceptions:
+                pkg_lic = exception.split(':')
+                pkg_exceptions[pkg_lic[0]] = pkg_lic[1]
+
+#            if any((pn in execption and incompatible_lic in exception) for execption in exceptions):
+            if any(execption.startswith(pn + ':') for execption in exceptions):
                 '''
-                We need to track what we are whitelisting and why. If pn is
-                incompatible we need to be able to note that the image that
-                is created may infact contain incompatible licenses despite
+                We need to track which recipes are in the exception
+                list and why. If pn is incompatible we need to be
+                able to note that the image that is created may
+                infact contain incompatible licenses despite
                 INCOMPATIBLE_LICENSE being set.
                 '''
-                bb.note("Including %s as buildable despite it having an incompatible license because it has been whitelisted" % pn)
+                bb.note("Including %s as a buildable recipe despite it having an incompatible license because it was found in the exception list" % pn)
             else:
                 pkgs = d.getVar('PACKAGES').split()
                 skipped_pkgs = {}
meta/classes/license_image.bbclass +10 -17
index bf70bee99bb..c6f04d30733 100644
@@ -54,28 +54,21 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
     bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
     bad_licenses = expand_wildcard_licenses(d, bad_licenses)
 
-    whitelist = []
-    for lic in bad_licenses:
-        whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-
+    exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
-            if bad_licenses and pkg not in whitelist:
-                try:
+            if bad_licenses and not any((pkg + ":") in execption for execption in exceptions):
                     licenses = incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"])
                     if licenses:
                         bb.fatal("Package %s cannot be installed into the image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses)))
-                    (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
-                        oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
-                        bad_licenses, canonical_license, d)
-                except oe.license.LicenseError as exc:
-                    bb.fatal('%s: %s' % (d.getVar('P'), exc))
-            else:
-                pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
-                pkg_dic[pkg]["LICENSES"] = re.sub(r'  *', ' ', pkg_dic[pkg]["LICENSES"])
-                pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
-                if pkg in whitelist:
-                    oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been whitelisted." %(pkg, pkg_dic[pkg]["LICENSE"]), d)
+            elif any((pkg + ":") in execption for execption in exceptions):
+                oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been allowed by exception list." %(pkg, pkg_dic[pkg]["LICENSE"]), d)
+            try:
+                (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
+                    oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
+                    bad_licenses, canonical_license, d)
+            except oe.license.LicenseError as exc:
+                bb.fatal('%s: %s' % (d.getVar('P'), exc))
 
             if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
                 # Rootfs manifest
meta/classes/multilib.bbclass +3 -3
index ec2013198ce..7d3b71c0d0a 100644
@@ -75,11 +75,11 @@ python multilib_virtclass_handler () {
     e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False))
     e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override)
 
-    # Expand WHITELIST_GPL-3.0 with multilib prefix
-    pkgs = e.data.getVar("WHITELIST_GPL-3.0")
+    # Expand INCOMPATIBLE_LICENSE_EXCEPTIONS with multilib prefix
+    pkgs = e.data.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS")
     for pkg in pkgs.split():
         pkgs += " " + variant + "-" + pkg
-    e.data.setVar("WHITELIST_GPL-3.0", pkgs)
+    e.data.setVar("INCOMPATIBLE_LICENSE_EXCEPTIONS", pkgs)
 
     # DEFAULTTUNE can change TARGET_ARCH override so expand this now before update_data
     newtune = e.data.getVar("DEFAULTTUNE:" + "virtclass-multilib-" + variant, False)
meta/conf/bitbake.conf +10 -0
index 6fb7bfeb23c..7f0de51fa7b 100644
@@ -110,6 +110,16 @@ BB_RENAMED_VARIABLES[INHERIT_BLACKLIST] = "is a deprecated variable and no longe
 BB_RENAMED_VARIABLES[TUNEABI_WHITELIST] = "is a deprecated variable and support has been removed"
 BB_RENAMED_VARIABLES[LICENSE_FLAGS_WHITELIST] = "LICENSE_FLAGS_ACCEPTED"
 
+BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0-only] = "INCOMPATIBLE_LICENSE_EXCEPTIONS"
+BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0-or-later] = "INCOMPATIBLE_LICENSE_EXCEPTIONS"
+BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0-only] = "INCOMPATIBLE_LICENSE_EXCEPTIONS"
+BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0-or-later] = "INCOMPATIBLE_LICENSE_EXCEPTIONS"
+
+# These are deprecated version and should be updated to approved names
+BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:GPL-3.0-only'"
+BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0+] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:GPL-3.0-or-later'"
+BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:LGPL-3.0-only'"
+BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0+] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:LGPL-3.0-or-later'"
 ##################################################################
 # Architecture-dependent build variables.
 ##################################################################
meta/conf/distro/include/default-distrovars.inc +1 -1
index 3bba651a776..dcc4a932a17 100644
@@ -20,7 +20,7 @@ DISTRO_FEATURES_DEFAULT ?= "acl alsa argp bluetooth debuginfod ext2 ipv4 ipv6 la
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT}"
 IMAGE_FEATURES ?= ""
 
-WHITELIST_GPL-3.0 ?= ""
+INCOMPATIBLE_LICENSE_EXCEPTIONS ?= ""
 
 COMMERCIAL_AUDIO_PLUGINS ?= ""
 # COMMERCIAL_AUDIO_PLUGINS ?= "gst-plugins-ugly-mad gst-plugins-ugly-mpegaudioparse"
meta/lib/oeqa/selftest/cases/incompatible_lic.py +5 -5
index fd3b3f409e3..08de3023c0a 100644
@@ -86,7 +86,7 @@ class IncompatibleLicensePerImageTests(OESelftestTestCase):
     def default_config(self):
         return """
 IMAGE_INSTALL:append = " bash"
-INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0"
+INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0-or-later LGPL-3.0"
 """
 
     def test_bash_default(self):
@@ -110,8 +110,8 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0"
 
         bitbake('core-image-minimal')
 
-    def test_bash_whitelist(self):
-        self.write_config(self.default_config() + '\nWHITELIST_GPL-3.0:pn-core-image-minimal = "bash"')
+    def test_bash_license_exceptions(self):
+        self.write_config(self.default_config() + '\nINCOMPATIBLE_LICENSE_EXCEPTIONS:pn-core-image-minimal = "bash:GPL-3.0-or-later"')
 
         bitbake('core-image-minimal')
 
@@ -125,8 +125,8 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0"
     def test_core_image_full_cmdline_weston(self):
         self.write_config("""
 INHERIT += "testimage"
-INCOMPATIBLE_LICENSE:pn-core-image-full-cmdline = "GPL-3.0 LGPL-3.0"
-INCOMPATIBLE_LICENSE:pn-core-image-weston = "GPL-3.0 LGPL-3.0"
+INCOMPATIBLE_LICENSE:pn-core-image-full-cmdline = "GPL-3.0-or-later LGPL-3.0"
+INCOMPATIBLE_LICENSE:pn-core-image-weston = "GPL-3.0-or-later LGPL-3.0"
 # Settings for full-cmdline
 RDEPENDS:packagegroup-core-full-cmdline-utils:remove = "bash bc coreutils cpio ed findutils gawk grep mc mc-fish mc-helpers mc-helpers-perl sed tar time"
 RDEPENDS:packagegroup-core-full-cmdline-dev-utils:remove = "diffutils m4 make patch"
Signature · 1 line
2.31.1
author
#1 lore raw

PATCH v3 2/2 INCOMPATIBLE_LICENSE: add has_pkg_license_exception()

This adds in the new function to check for both package and license are in the new INCOMPATIBLE_LICENSE_EXCEPTION list.

This has been tested by changing the skeleton/hello to MIT-X and using that license to verify it will be skipped or not installed. oe-selftest was also used.

---
 meta/classes/base.bbclass          | 66 ++++++++++++------------------
 meta/classes/license_image.bbclass |  5 ++-
 meta/lib/oe/license.py             | 10 +++++
 3 files changed, 40 insertions(+), 41 deletions(-)
meta/classes/base.bbclass +27 -39
index ddca87d4a8c..fccf3df17ff 100644
@@ -597,46 +597,34 @@ python () {
 
             exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
 
-            pkg_exceptions = {}
-            for exception in exceptions:
-                pkg_lic = exception.split(':')
-                pkg_exceptions[pkg_lic[0]] = pkg_lic[1]
-
-#            if any((pn in execption and incompatible_lic in exception) for execption in exceptions):
-            if any(execption.startswith(pn + ':') for execption in exceptions):
-                '''
-                We need to track which recipes are in the exception
-                list and why. If pn is incompatible we need to be
-                able to note that the image that is created may
-                infact contain incompatible licenses despite
-                INCOMPATIBLE_LICENSE being set.
-                '''
-                bb.note("Including %s as a buildable recipe despite it having an incompatible license because it was found in the exception list" % pn)
-            else:
-                pkgs = d.getVar('PACKAGES').split()
-                skipped_pkgs = {}
-                unskipped_pkgs = []
-                for pkg in pkgs:
-                    incompatible_lic = incompatible_license(d, bad_licenses, pkg)
-                    if incompatible_lic:
-                        skipped_pkgs[pkg] = incompatible_lic
-                    else:
-                        unskipped_pkgs.append(pkg)
-                if unskipped_pkgs:
-                    for pkg in skipped_pkgs:
-                        bb.debug(1, "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
-                        d.setVar('_exclude_incompatible-' + pkg, ' '.join(skipped_pkgs[pkg]))
-                    for pkg in unskipped_pkgs:
-                        bb.debug(1, "Including the package %s" % pkg)
+            pkgs = d.getVar('PACKAGES').split()
+            skipped_pkgs = {}
+            unskipped_pkgs = []
+            for pkg in pkgs:
+                pkg_exception = oe.license.has_pkg_license_exception(pkg, bad_licenses, exceptions)
+
+                incompatible_lic = incompatible_license(d, bad_licenses, pkg)
+                if incompatible_lic and not pkg_exception:
+                    skipped_pkgs[pkg] = incompatible_lic
                 else:
-                    incompatible_lic = incompatible_license(d, bad_licenses)
-                    for pkg in skipped_pkgs:
-                        incompatible_lic += skipped_pkgs[pkg]
-                    incompatible_lic = sorted(list(set(incompatible_lic)))
-
-                    if incompatible_lic:
-                        bb.debug(1, "Skipping recipe %s because of incompatible license(s): %s" % (pn, ' '.join(incompatible_lic)))
-                        raise bb.parse.SkipRecipe("it has incompatible license(s): %s" % ' '.join(incompatible_lic))
+                    unskipped_pkgs.append(pkg)
+
+            if unskipped_pkgs:
+                for pkg in skipped_pkgs:
+                    bb.warn( "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
+                    bb.debug(1, "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
+                    d.setVar('_exclude_incompatible-' + pkg, ' '.join(skipped_pkgs[pkg]))
+                for pkg in unskipped_pkgs:
+                    bb.debug(1, "Including the package %s" % pkg)
+            else:
+                incompatible_lic = incompatible_license(d, bad_licenses)
+                for pkg in skipped_pkgs:
+                    incompatible_lic += skipped_pkgs[pkg]
+                incompatible_lic = sorted(list(set(incompatible_lic)))
+
+                if incompatible_lic:
+                    bb.warn( "Skipping recipe %s because of incompatible license(s): %s" % (pn, ' '.join(incompatible_lic)))
+                    raise bb.parse.SkipRecipe("it has incompatible license(s): %s" % ' '.join(incompatible_lic))
 
     needsrcrev = False
     srcuri = d.getVar('SRC_URI')
meta/classes/license_image.bbclass +3 -2
index c6f04d30733..cae094126db 100644
@@ -57,11 +57,12 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
     exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
-            if bad_licenses and not any((pkg + ":") in execption for execption in exceptions):
+            pkg_exception = oe.license.has_pkg_license_exception(pkg, bad_licenses, exceptions)
+            if bad_licenses and not pkg_exception:
                     licenses = incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"])
                     if licenses:
                         bb.fatal("Package %s cannot be installed into the image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses)))
-            elif any((pkg + ":") in execption for execption in exceptions):
+            elif pkg_exception:
                 oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been allowed by exception list." %(pkg, pkg_dic[pkg]["LICENSE"]), d)
             try:
                 (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
meta/lib/oe/license.py +10 -0
index 8955cbdeb24..2404bbecb18 100644
@@ -242,3 +242,13 @@ def list_licenses(licensestr):
     except SyntaxError as exc:
         raise LicenseSyntaxError(licensestr, exc)
     return visitor.licenses
+
+def has_pkg_license_exception(pkg, dont_want_licenses, exceptions):
+    """Check the exception list for the package and return a boolean"""
+
+    pkg_exception = False
+    for bad_lic in dont_want_licenses:
+        if (pkg + ':' + bad_lic) in exceptions:
+            pkg_exception = True
+    return pkg_exception
+
Signature · 1 line
2.31.1
author
#2 lore raw

PATCH v3 2/2 license.py: rename variales

In reply to Saul Wold
---
 meta/lib/oe/license.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)
meta/lib/oe/license.py +15 -13
index b5d378a549b..4cd382b4fd7 100644
@@ -99,20 +99,22 @@ def flattened_licenses(licensestr, choose_licenses):
         raise LicenseSyntaxError(licensestr, exc)
     return flatten.licenses
 
-def is_included(licensestr, whitelist=None, blacklist=None):
-    """Given a license string and whitelist and blacklist, determine if the
-    license string matches the whitelist and does not match the blacklist.
-
-    Returns a tuple holding the boolean state and a list of the applicable
-    licenses that were excluded if state is False, or the licenses that were
-    included if the state is True.
+def is_included(licensestr, include=None, exclude=None):
+    """Given a license string and include list and exclude list,
+        determine if the license string matches the an included
+        license and does dont match an excluded license.
+
+        Returns a tuple holding the boolean state and a list of
+        the applicable licenses that were excluded if state is
+        False, or the licenses that were included if the state
+        is True.
     """
 
     def include_license(license):
-        return any(fnmatch(license, pattern) for pattern in whitelist)
+        return any(fnmatch(license, pattern) for pattern in include)
 
     def exclude_license(license):
-        return any(fnmatch(license, pattern) for pattern in blacklist)
+        return any(fnmatch(license, pattern) for pattern in exclude)
 
     def choose_licenses(alpha, beta):
         """Select the option in an OR which is the 'best' (has the most
@@ -131,11 +133,11 @@ def is_included(licensestr, whitelist=None, blacklist=None):
         else:
             return beta
 
-    if not whitelist:
-        whitelist = ['*']
+    if not include:
+        include = ['*']
 
-    if not blacklist:
-        blacklist = []
+    if not exclude:
+        exclude = []
 
     licenses = flattened_licenses(licensestr, choose_licenses)
     excluded = [lic for lic in licenses if exclude_license(lic)]
Signature · 1 line
2.31.1