INCOMPATIBLE_LICENSE re-work
PATCH v3 1/2 INCOMPATIBLE_LICENSE re-work
From: Saul Wold <[email protected]>
This re-writes the INCOMPATIBLE_LICENSE checking code to replace the WHITELIST_<lic> with INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:<lic> <pkg>:<lic> ...'
This initial set of changes leaves most of the code structure in place, but the code in base.bbclass needs to be re-written to make the check more consistent around packages (PKGS) and not recipe names (PN). This also is taking into account the changes for SPDX licenses.
--- meta/classes/base.bbclass | 26 +++++++++--------- meta/classes/license_image.bbclass | 27 +++++++------------ meta/classes/multilib.bbclass | 6 ++--- meta/conf/bitbake.conf | 10 +++++++ .../distro/include/default-distrovars.inc | 2 +- .../oeqa/selftest/cases/incompatible_lic.py | 10 +++---- 6 files changed, 43 insertions(+), 38 deletions(-)
meta/classes/base.bbclass +14 -12
@@ -595,21 +595,23 @@ python () { if check_license and bad_licenses: bad_licenses = expand_wildcard_licenses(d, bad_licenses) - whitelist = [] - for lic in bad_licenses: - spdx_license = return_spdx(d, lic) - whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split()) - if spdx_license: - whitelist.extend((d.getVar("WHITELIST_" + spdx_license) or "").split()) - - if pn in whitelist: + exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() + + pkg_exceptions = {} + for exception in exceptions: + pkg_lic = exception.split(':') + pkg_exceptions[pkg_lic[0]] = pkg_lic[1] + +# if any((pn in execption and incompatible_lic in exception) for execption in exceptions): + if any(execption.startswith(pn + ':') for execption in exceptions): ''' - We need to track what we are whitelisting and why. If pn is - incompatible we need to be able to note that the image that - is created may infact contain incompatible licenses despite + We need to track which recipes are in the exception + list and why. If pn is incompatible we need to be + able to note that the image that is created may + infact contain incompatible licenses despite INCOMPATIBLE_LICENSE being set. ''' - bb.note("Including %s as buildable despite it having an incompatible license because it has been whitelisted" % pn) + bb.note("Including %s as a buildable recipe despite it having an incompatible license because it was found in the exception list" % pn) else: pkgs = d.getVar('PACKAGES').split() skipped_pkgs = {}
meta/classes/license_image.bbclass +10 -17
@@ -54,28 +54,21 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True): bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split() bad_licenses = expand_wildcard_licenses(d, bad_licenses) - whitelist = [] - for lic in bad_licenses: - whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split()) - + exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() with open(license_manifest, "w") as license_file: for pkg in sorted(pkg_dic): - if bad_licenses and pkg not in whitelist: - try: + if bad_licenses and not any((pkg + ":") in execption for execption in exceptions): licenses = incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"]) if licenses: bb.fatal("Package %s cannot be installed into the image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses))) - (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \ - oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"], - bad_licenses, canonical_license, d) - except oe.license.LicenseError as exc: - bb.fatal('%s: %s' % (d.getVar('P'), exc)) - else: - pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ', pkg_dic[pkg]["LICENSE"]) - pkg_dic[pkg]["LICENSES"] = re.sub(r' *', ' ', pkg_dic[pkg]["LICENSES"]) - pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split() - if pkg in whitelist: - oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been whitelisted." %(pkg, pkg_dic[pkg]["LICENSE"]), d) + elif any((pkg + ":") in execption for execption in exceptions): + oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been allowed by exception list." %(pkg, pkg_dic[pkg]["LICENSE"]), d) + try: + (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \ + oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"], + bad_licenses, canonical_license, d) + except oe.license.LicenseError as exc: + bb.fatal('%s: %s' % (d.getVar('P'), exc)) if not "IMAGE_MANIFEST" in pkg_dic[pkg]: # Rootfs manifest
meta/classes/multilib.bbclass +3 -3
@@ -75,11 +75,11 @@ python multilib_virtclass_handler () { e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False)) e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override) - # Expand WHITELIST_GPL-3.0 with multilib prefix - pkgs = e.data.getVar("WHITELIST_GPL-3.0") + # Expand INCOMPATIBLE_LICENSE_EXCEPTIONS with multilib prefix + pkgs = e.data.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") for pkg in pkgs.split(): pkgs += " " + variant + "-" + pkg - e.data.setVar("WHITELIST_GPL-3.0", pkgs) + e.data.setVar("INCOMPATIBLE_LICENSE_EXCEPTIONS", pkgs) # DEFAULTTUNE can change TARGET_ARCH override so expand this now before update_data newtune = e.data.getVar("DEFAULTTUNE:" + "virtclass-multilib-" + variant, False)
meta/conf/bitbake.conf +10 -0
@@ -110,6 +110,16 @@ BB_RENAMED_VARIABLES[INHERIT_BLACKLIST] = "is a deprecated variable and no longe BB_RENAMED_VARIABLES[TUNEABI_WHITELIST] = "is a deprecated variable and support has been removed" BB_RENAMED_VARIABLES[LICENSE_FLAGS_WHITELIST] = "LICENSE_FLAGS_ACCEPTED" +BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0-only] = "INCOMPATIBLE_LICENSE_EXCEPTIONS" +BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0-or-later] = "INCOMPATIBLE_LICENSE_EXCEPTIONS" +BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0-only] = "INCOMPATIBLE_LICENSE_EXCEPTIONS" +BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0-or-later] = "INCOMPATIBLE_LICENSE_EXCEPTIONS" + +# These are deprecated version and should be updated to approved names +BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:GPL-3.0-only'" +BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0+] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:GPL-3.0-or-later'" +BB_RENAMED_VARIABLES[WHITELIST_GPL-3.0] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:LGPL-3.0-only'" +BB_RENAMED_VARIABLES[WHITELIST_LGPL-3.0+] = "is deprecated, convert to INCOMPATIBLE_LICENSE_EXCEPTIONS = '<pkg>:LGPL-3.0-or-later'" ################################################################## # Architecture-dependent build variables. ##################################################################
meta/conf/distro/include/default-distrovars.inc +1 -1
@@ -20,7 +20,7 @@ DISTRO_FEATURES_DEFAULT ?= "acl alsa argp bluetooth debuginfod ext2 ipv4 ipv6 la DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT}" IMAGE_FEATURES ?= "" -WHITELIST_GPL-3.0 ?= "" +INCOMPATIBLE_LICENSE_EXCEPTIONS ?= "" COMMERCIAL_AUDIO_PLUGINS ?= "" # COMMERCIAL_AUDIO_PLUGINS ?= "gst-plugins-ugly-mad gst-plugins-ugly-mpegaudioparse"
meta/lib/oeqa/selftest/cases/incompatible_lic.py +5 -5
@@ -86,7 +86,7 @@ class IncompatibleLicensePerImageTests(OESelftestTestCase): def default_config(self): return """ IMAGE_INSTALL:append = " bash" -INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0" +INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0-or-later LGPL-3.0" """ def test_bash_default(self):
@@ -110,8 +110,8 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0" bitbake('core-image-minimal') - def test_bash_whitelist(self): - self.write_config(self.default_config() + '\nWHITELIST_GPL-3.0:pn-core-image-minimal = "bash"') + def test_bash_license_exceptions(self): + self.write_config(self.default_config() + '\nINCOMPATIBLE_LICENSE_EXCEPTIONS:pn-core-image-minimal = "bash:GPL-3.0-or-later"') bitbake('core-image-minimal')
@@ -125,8 +125,8 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0 LGPL-3.0" def test_core_image_full_cmdline_weston(self): self.write_config(""" INHERIT += "testimage" -INCOMPATIBLE_LICENSE:pn-core-image-full-cmdline = "GPL-3.0 LGPL-3.0" -INCOMPATIBLE_LICENSE:pn-core-image-weston = "GPL-3.0 LGPL-3.0" +INCOMPATIBLE_LICENSE:pn-core-image-full-cmdline = "GPL-3.0-or-later LGPL-3.0" +INCOMPATIBLE_LICENSE:pn-core-image-weston = "GPL-3.0-or-later LGPL-3.0" # Settings for full-cmdline RDEPENDS:packagegroup-core-full-cmdline-utils:remove = "bash bc coreutils cpio ed findutils gawk grep mc mc-fish mc-helpers mc-helpers-perl sed tar time" RDEPENDS:packagegroup-core-full-cmdline-dev-utils:remove = "diffutils m4 make patch"
Signature · 1 line
2.31.1
PATCH v3 2/2 INCOMPATIBLE_LICENSE: add has_pkg_license_exception()
This adds in the new function to check for both package and license are in the new INCOMPATIBLE_LICENSE_EXCEPTION list.
This has been tested by changing the skeleton/hello to MIT-X and using that license to verify it will be skipped or not installed. oe-selftest was also used.
--- meta/classes/base.bbclass | 66 ++++++++++++------------------ meta/classes/license_image.bbclass | 5 ++- meta/lib/oe/license.py | 10 +++++ 3 files changed, 40 insertions(+), 41 deletions(-)
meta/classes/base.bbclass +27 -39
@@ -597,46 +597,34 @@ python () { exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() - pkg_exceptions = {} - for exception in exceptions: - pkg_lic = exception.split(':') - pkg_exceptions[pkg_lic[0]] = pkg_lic[1] - -# if any((pn in execption and incompatible_lic in exception) for execption in exceptions): - if any(execption.startswith(pn + ':') for execption in exceptions): - ''' - We need to track which recipes are in the exception - list and why. If pn is incompatible we need to be - able to note that the image that is created may - infact contain incompatible licenses despite - INCOMPATIBLE_LICENSE being set. - ''' - bb.note("Including %s as a buildable recipe despite it having an incompatible license because it was found in the exception list" % pn) - else: - pkgs = d.getVar('PACKAGES').split() - skipped_pkgs = {} - unskipped_pkgs = [] - for pkg in pkgs: - incompatible_lic = incompatible_license(d, bad_licenses, pkg) - if incompatible_lic: - skipped_pkgs[pkg] = incompatible_lic - else: - unskipped_pkgs.append(pkg) - if unskipped_pkgs: - for pkg in skipped_pkgs: - bb.debug(1, "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg]))) - d.setVar('_exclude_incompatible-' + pkg, ' '.join(skipped_pkgs[pkg])) - for pkg in unskipped_pkgs: - bb.debug(1, "Including the package %s" % pkg) + pkgs = d.getVar('PACKAGES').split() + skipped_pkgs = {} + unskipped_pkgs = [] + for pkg in pkgs: + pkg_exception = oe.license.has_pkg_license_exception(pkg, bad_licenses, exceptions) + + incompatible_lic = incompatible_license(d, bad_licenses, pkg) + if incompatible_lic and not pkg_exception: + skipped_pkgs[pkg] = incompatible_lic else: - incompatible_lic = incompatible_license(d, bad_licenses) - for pkg in skipped_pkgs: - incompatible_lic += skipped_pkgs[pkg] - incompatible_lic = sorted(list(set(incompatible_lic))) - - if incompatible_lic: - bb.debug(1, "Skipping recipe %s because of incompatible license(s): %s" % (pn, ' '.join(incompatible_lic))) - raise bb.parse.SkipRecipe("it has incompatible license(s): %s" % ' '.join(incompatible_lic)) + unskipped_pkgs.append(pkg) + + if unskipped_pkgs: + for pkg in skipped_pkgs: + bb.warn( "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg]))) + bb.debug(1, "Skipping the package %s at do_rootfs because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg]))) + d.setVar('_exclude_incompatible-' + pkg, ' '.join(skipped_pkgs[pkg])) + for pkg in unskipped_pkgs: + bb.debug(1, "Including the package %s" % pkg) + else: + incompatible_lic = incompatible_license(d, bad_licenses) + for pkg in skipped_pkgs: + incompatible_lic += skipped_pkgs[pkg] + incompatible_lic = sorted(list(set(incompatible_lic))) + + if incompatible_lic: + bb.warn( "Skipping recipe %s because of incompatible license(s): %s" % (pn, ' '.join(incompatible_lic))) + raise bb.parse.SkipRecipe("it has incompatible license(s): %s" % ' '.join(incompatible_lic)) needsrcrev = False srcuri = d.getVar('SRC_URI')
meta/classes/license_image.bbclass +3 -2
@@ -57,11 +57,12 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True): exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() with open(license_manifest, "w") as license_file: for pkg in sorted(pkg_dic): - if bad_licenses and not any((pkg + ":") in execption for execption in exceptions): + pkg_exception = oe.license.has_pkg_license_exception(pkg, bad_licenses, exceptions) + if bad_licenses and not pkg_exception: licenses = incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"]) if licenses: bb.fatal("Package %s cannot be installed into the image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses))) - elif any((pkg + ":") in execption for execption in exceptions): + elif pkg_exception: oe.qa.handle_error('license-incompatible', "Including %s with an incompatible license %s into the image, because it has been allowed by exception list." %(pkg, pkg_dic[pkg]["LICENSE"]), d) try: (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
meta/lib/oe/license.py +10 -0
@@ -242,3 +242,13 @@ def list_licenses(licensestr): except SyntaxError as exc: raise LicenseSyntaxError(licensestr, exc) return visitor.licenses + +def has_pkg_license_exception(pkg, dont_want_licenses, exceptions): + """Check the exception list for the package and return a boolean""" + + pkg_exception = False + for bad_lic in dont_want_licenses: + if (pkg + ':' + bad_lic) in exceptions: + pkg_exception = True + return pkg_exception +
Signature · 1 line
2.31.1
PATCH v3 2/2 license.py: rename variales
In reply to Saul Wold--- meta/lib/oe/license.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-)
meta/lib/oe/license.py +15 -13
@@ -99,20 +99,22 @@ def flattened_licenses(licensestr, choose_licenses): raise LicenseSyntaxError(licensestr, exc) return flatten.licenses -def is_included(licensestr, whitelist=None, blacklist=None): - """Given a license string and whitelist and blacklist, determine if the - license string matches the whitelist and does not match the blacklist. - - Returns a tuple holding the boolean state and a list of the applicable - licenses that were excluded if state is False, or the licenses that were - included if the state is True. +def is_included(licensestr, include=None, exclude=None): + """Given a license string and include list and exclude list, + determine if the license string matches the an included + license and does dont match an excluded license. + + Returns a tuple holding the boolean state and a list of + the applicable licenses that were excluded if state is + False, or the licenses that were included if the state + is True. """ def include_license(license): - return any(fnmatch(license, pattern) for pattern in whitelist) + return any(fnmatch(license, pattern) for pattern in include) def exclude_license(license): - return any(fnmatch(license, pattern) for pattern in blacklist) + return any(fnmatch(license, pattern) for pattern in exclude) def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most
@@ -131,11 +133,11 @@ def is_included(licensestr, whitelist=None, blacklist=None): else: return beta - if not whitelist: - whitelist = ['*'] + if not include: + include = ['*'] - if not blacklist: - blacklist = [] + if not exclude: + exclude = [] licenses = flattened_licenses(licensestr, choose_licenses) excluded = [lic for lic in licenses if exclude_license(lic)]
Signature · 1 line
2.31.1